[JENKINS-10511] updated the backend that builds JDK listing.

jenkins-infra · Jul 29, 2011 · 729782d · 729782d
1 parent 36705b0
commit 729782d
Show file tree

Hide file tree

Showing 5 changed files with 128 additions and 181 deletions.
diff --git a/list-ant/pom.xml b/list-ant/pom.xml
@@ -1,8 +1,9 @@
-<?xml version="1.0"?><project>
+<?xml version="1.0"?>
+<project>
   <parent>
     <artifactId>crawler</artifactId>
-    <groupId>org.jvnet.hudson</groupId>
-    <version>1.7</version>
+    <groupId>org.jenkins-ci</groupId>
+    <version>1.0-SNAPSHOT</version>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>list-ant</artifactId>

diff --git a/list-jdk/pom.xml b/list-jdk/pom.xml
@@ -1,23 +1,28 @@
-<?xml version="1.0"?><project>
+<?xml version="1.0"?>
+<project>
   <parent>
     <artifactId>crawler</artifactId>
-    <groupId>org.jvnet.hudson</groupId>
-    <version>1.7</version>
+    <groupId>org.jenkins-ci</groupId>
+    <version>1.0-SNAPSHOT</version>
+    <relativePath>../pom.xml</relativePath>
   </parent>
   <modelVersion>4.0.0</modelVersion>
   <artifactId>list-jdk</artifactId>
+
   <name>List JDK Installations</name>
   <version>1.0-SNAPSHOT</version>
   <description>
-    This runs on http://hudson.sfbay/job/hudson-crawler/ regularly to publish
-    the list of JDKs to https://hudson.dev.java.net/jdk.json
+    Builds the list of JDKs.
   </description>
 
   <build>
     <plugins>
       <plugin>
-        <groupId>org.codehaus.groovy.maven</groupId>
-        <artifactId>gmaven-plugin</artifactId>
+        <groupId>org.codehaus.mojo</groupId>
+        <artifactId>exec-maven-plugin</artifactId>
+        <configuration>
+          <mainClass>ListJDK</mainClass>
+        </configuration>
       </plugin>
     </plugins>
   </build>

diff --git a/list-jdk/src/main/groovy/App.groovy b/list-jdk/src/main/groovy/App.groovy
diff --git a/list-jdk/src/main/java/ListJDK.java b/list-jdk/src/main/java/ListJDK.java
@@ -0,0 +1,98 @@
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import net.sf.json.JSONArray;
+import net.sf.json.JSONObject;
+import net.sourceforge.htmlunit.corejs.javascript.IdScriptableObject;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.Set;
+
+/**
+ * Couldn't get the Groovy version working because of org/w3c/dom/UserDataHandler.class in Jaxen
+ * that makes Groovy classloader unhappy.
+ *
+ * @author Kohsuke Kawaguchi
+ */
+public class ListJDK {
+    private final WebClient wc;
+
+    public ListJDK() {
+        wc = new WebClient();
+        wc.setCssEnabled(false);
+    }
+
+    public static void main(String[] args) throws Exception {
+        OutputStreamWriter w = new OutputStreamWriter(new FileOutputStream("jdk.json"), "UTF-8");
+        w.write(new ListJDK().build().toString());
+        w.close();
+    }
+
+    private JSONObject build() throws IOException {
+        return new JSONObject()
+            .element("version", 2)
+            .element("data",new JSONArray()
+                .element(family("JDK 7", parse("http://www.oracle.com/technetwork/java/javase/downloads/java-se-jdk-7-download-432154.html")))
+                .element(family("JDK 6", parse("http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase6-419409.html")))
+                .element(family("JDK 5", parse("http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase5-419410.html")))
+                .element(family("JDK 1.4", parse("http://www.oracle.com/technetwork/java/javasebusiness/downloads/java-archive-downloads-javase14-419411.html"))));
+    }
+
+    /**
+     * Builds a data structure for a major release.
+     */
+    private JSONObject family(String name, JSONObject data) {
+        JSONObject o = new JSONObject();
+        o.put("name",name);
+        JSONArray releases = new JSONArray();
+        for (String n : (Set<String>)data.keySet()) {
+            releases.add(release(n,data.getJSONObject(n)));
+        }
+        o.put("releases",releases);
+        return o;
+    }
+
+    private JSONObject release(String name, JSONObject src) {
+        JSONObject in = src.getJSONObject("files");
+        JSONArray files = new JSONArray();
+        for (String n : (Set<String>)in.keySet()) {
+            files.add(file(n, in.getJSONObject(n)));
+        }
+        return src.element("name",name).element("files",files);
+    }
+
+    private JSONObject file(String name, JSONObject src) {
+        return src.element("name",name).element("size",(Object)null);
+    }
+
+    private JSONObject parse(String url) throws IOException {
+        HtmlPage p = getPage(url);
+
+        return (JSONObject)toJSON(p.executeJavaScript("downloads").getJavaScriptResult());
+    }
+
+    private Object toJSON(Object o) {
+        // the code in the page instantiates array but uses it as an object,
+        // so we convert them all into JSONObject
+        if (o instanceof IdScriptableObject) {
+            IdScriptableObject na = (IdScriptableObject) o;
+            JSONObject ja = new JSONObject();
+            for (Object key : na.getIds()) {
+                ja.put(key.toString(), toJSON(na.get(key)));
+            }
+            return ja;
+        }
+        // primitives
+        return o;
+    }
+
+    private HtmlPage getPage(String url) throws IOException {
+//        System.out.println("Fetching url} ...")
+        long start = System.currentTimeMillis();
+        HtmlPage p = wc.getPage(url);
+        long end = System.currentTimeMillis();
+//        println("done (took ${end - start} msec)")
+        return p;
+    }
+}
diff --git a/pom.xml b/pom.xml
@@ -4,11 +4,12 @@
   <parent>
     <groupId>org.jenkins-ci</groupId>
     <artifactId>jenkins</artifactId>
-    <version>1.17</version>
+    <version>1.21</version>
   </parent>
 
   <artifactId>crawler</artifactId>
   <name>Tool Crawler</name>
+  <version>1.0-SNAPSHOT</version>
   <packaging>pom</packaging>
   <description>Code that we use behind the scene.</description>
 
@@ -19,14 +20,21 @@
 
   <dependencies>
     <dependency>
-      <groupId>org.jvnet.hudson</groupId>
+      <groupId>org.jenkins-ci</groupId>
       <artifactId>htmlunit</artifactId>
-      <version>2.2-hudson-9</version>
+      <version>2.6-jenkins-4</version>
+      <exclusions>
+        <exclusion>
+          <!--  hides JDK DOM classes in Eclipse -->
+          <groupId>xml-apis</groupId>
+          <artifactId>xml-apis</artifactId>
+        </exclusion>
+      </exclusions>
     </dependency>
     <dependency>
-      <groupId>org.jvnet.hudson.main</groupId>
-      <artifactId>hudson-core</artifactId>
-      <version>1.300</version>
+      <groupId>org.jenkins-ci.main</groupId>
+      <artifactId>jenkins-core</artifactId>
+      <version>1.409</version>
     </dependency>
     <dependency>
       <groupId>javax.servlet</groupId>