[JENKINS-32328] process multiple update-centers for ToolInstallers

jenkinsci · Jan 22, 2016 · 80a6a9e · ndeloof · Feb 18, 2016 · batmat
1 parent 7e05b50
commit 80a6a9e
Show file tree

Hide file tree

Showing 17 changed files with 836 additions and 27 deletions.
diff --git a/core/src/main/java/hudson/model/DownloadService.java b/core/src/main/java/hudson/model/DownloadService.java
@@ -38,8 +38,11 @@
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.reflect.Field;
 import java.net.URL;
 import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import jenkins.model.DownloadSettings;
@@ -301,6 +304,24 @@ public String getUrl() {
             return Jenkins.getInstance().getUpdateCenter().getDefaultBaseUrl()+"updates/"+url;
         }
 
+        /**
+         * URLs to download from.
+         */
+        public List<String> getUrls() {
+            List<String> updateSites = new ArrayList<String>();
+            for (UpdateSite site : Jenkins.getActiveInstance().getUpdateCenter().getSiteList()) {
+                String siteUrl = site.getUrl();
+                int baseUrlEnd = siteUrl.indexOf("update-center.json");
+                if (baseUrlEnd != -1) {
+                    String siteBaseUrl = siteUrl.substring(0, baseUrlEnd);
+                    updateSites.add(siteBaseUrl + "updates/" + url);
+                } else {
+                    LOGGER.log(Level.WARNING, "Url {0} does not look like an update center:", siteUrl);
+                }
+            }
+            return updateSites;
+        }
+
         /**
          * How often do we retrieve the new image?
          *
@@ -364,15 +385,6 @@ public void doPostBack(StaplerRequest req, StaplerResponse rsp) throws IOExcepti
         }
 
         private FormValidation load(String json, long dataTimestamp) throws IOException {
-            JSONObject o = JSONObject.fromObject(json);
-
-            if (signatureCheck) {
-                FormValidation e = new JSONSignatureValidator("downloadable '"+id+"'").verifySignature(o);
-                if (e.kind!= Kind.OK) {
-                    return e;
-                }
-            }
-
             TextFile df = getDataFile();
             df.write(json);
             df.file.setLastModified(dataTimestamp);
@@ -382,7 +394,72 @@ private FormValidation load(String json, long dataTimestamp) throws IOException
 
         @Restricted(NoExternalUse.class)
         public FormValidation updateNow() throws IOException {
-            return load(loadJSONHTML(new URL(getUrl() + ".html?id=" + URLEncoder.encode(getId(), "UTF-8") + "&version=" + URLEncoder.encode(Jenkins.VERSION, "UTF-8"))), System.currentTimeMillis());
+            List<JSONObject> jsonList = new ArrayList<>();
+            for (String site : getUrls()) {
+                String jsonString;
+                try {
+                    jsonString = loadJSONHTML(new URL(site + ".html?id=" + URLEncoder.encode(getId(), "UTF-8") + "&version=" + URLEncoder.encode(Jenkins.VERSION, "UTF-8")));
+                } catch (Exception e) {
+                    LOGGER.log(Level.WARNING, "Could not load json from " + site, e );
+                    continue;
+                }
+                JSONObject o = JSONObject.fromObject(jsonString);
+                if (signatureCheck) {
+                    FormValidation e = new JSONSignatureValidator("downloadable '"+id+"'").verifySignature(o);
+                    if (e.kind!= Kind.OK) {
+                        continue;
+                    }
+                }
+                jsonList.add(o);
+            }
+            if (jsonList.size() == 0) {
+                return FormValidation.warning("None of the Update Sites passed the signature check");
+            }
+            JSONObject reducedJson = reduce(jsonList);
+            return load(reducedJson.toString(), System.currentTimeMillis());
+        }
+
+        /**
+         * Function that takes multiple JSONObjects and returns a single one.
+         * @param jsonList to be processed
+         * @return a single JSONObject
+         */
+        public JSONObject reduce(List<JSONObject> jsonList) {
+            return jsonList.get(0);
+        }
+
+        /**
+         * check if the list of update center entries has duplicates
+         * @param genericList list of entries coming from multiple update centers
+         * @param comparator the unique ID of an entry
+         * @param <T> the generic class
+         * @return true if the list has duplicates, false otherwise
+         */
+        public static <T> boolean hasDuplicates (List<T> genericList, String comparator) {
+            if (genericList.isEmpty()) {
+                return false;
+            }
+            Field field;
+            try {
+                field = genericList.get(0).getClass().getDeclaredField(comparator);
+            } catch (NoSuchFieldException e) {
+                LOGGER.warning("comparator: " + comparator + "does not exist for " + genericList.get(0).getClass() + ", " + e);
+                return false;
+            }
+            for (int i = 0; i < genericList.size(); i ++ ) {
+                T data1 = genericList.get(i);
+                for (int j = i + 1; j < genericList.size(); j ++ ) {
+                    T data2 = genericList.get(j);
+                    try {
+                        if (field.get(data1).equals(field.get(data2))) {
+                            return true;
+                        }
+                    } catch (IllegalAccessException e) {
+                        LOGGER.warning("could not access field: " + comparator + ", " + e);
+                    }
+                }
+            }
+            return false;
         }
 
         /**

diff --git a/core/src/main/java/hudson/tools/DownloadFromUrlInstaller.java b/core/src/main/java/hudson/tools/DownloadFromUrlInstaller.java
@@ -10,6 +10,7 @@
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 import java.net.URL;
 
@@ -126,10 +127,68 @@ protected DescriptorImpl() {
             Downloadable.all().add(createDownloadable());
         }
 
-        protected Downloadable createDownloadable() {
+        /**
+         * function that creates a {@link Downloadable}.
+         * @return a downloadable object
+         */
+        public Downloadable createDownloadable() {
+            if (this instanceof DownloadFromUrlInstaller.DescriptorImpl) {
+                final DownloadFromUrlInstaller.DescriptorImpl delegate = (DownloadFromUrlInstaller.DescriptorImpl)this;
+                return new Downloadable(getId()) {
+                    public JSONObject reduce(List<JSONObject> jsonList) {
+                        return delegate.reduce(jsonList);
+                    }
+                };
+            }
             return new Downloadable(getId());
         }
 
+        private JSONObject reduce(List<JSONObject> jsonList) {
+            List<ToolInstallerEntry> reducedToolEntries = new LinkedList<>();
+            //collect all tool installers objects from the multiple json objects
+            for (JSONObject jsonToolList : jsonList) {
+                ToolInstallerList toolInstallerList = (ToolInstallerList) JSONObject.toBean(jsonToolList, ToolInstallerList.class);
+                reducedToolEntries.addAll(Arrays.asList(toolInstallerList.list));
+            }
+
+            while (Downloadable.hasDuplicates(reducedToolEntries, "id")) {
+                List<ToolInstallerEntry> tmpToolInstallerEntries = new LinkedList<>();
+                //we need to skip the processed entries
+                boolean processed[] = new boolean[reducedToolEntries.size()];
+                for (int i = 0; i < reducedToolEntries.size(); i++) {
+                    if (processed[i] == true) {
+                        continue;
+                    }
+                    ToolInstallerEntry data1 = reducedToolEntries.get(i);
+                    boolean hasDuplicate = false;
+                    for (int j = i + 1; j < reducedToolEntries.size(); j ++) {
+                        ToolInstallerEntry data2 = reducedToolEntries.get(j);
+                        //if we found a duplicate we choose the first one
+                        if (data1.id.equals(data2.id)) {
+                            hasDuplicate = true;
+                            processed[j] = true;
+                            tmpToolInstallerEntries.add(data1);
+                            //after the first duplicate has been found we break the loop since the duplicates are
+                            //processed two by two
+                            break;
+                        }
+                    }
+                    //if no duplicate has been found we just insert the entry in the tmp list
+                    if (!hasDuplicate) {
+                        tmpToolInstallerEntries.add(data1);
+                    }
+                }
+                reducedToolEntries = tmpToolInstallerEntries;
+            }
+
+            ToolInstallerList toolInstallerList = new ToolInstallerList();
+            toolInstallerList.list = new ToolInstallerEntry[reducedToolEntries.size()];
+            reducedToolEntries.toArray(toolInstallerList.list);
+            JSONObject reducedToolEntriesJsonList = JSONObject.fromObject(toolInstallerList);
+            //return the list with no duplicates
+            return reducedToolEntriesJsonList;
+        }
+
         /**
          * This ID needs to be unique, and needs to match the ID token in the JSON update file.
          * <p>