[FIXED JENKINS-18289]

The code is smart enough to compute a minimum set of builds to remove to restore order consistency.
jenkinsci · Jul 30, 2013 · d90d47d · d90d47d
1 parent a982154
commit d90d47d
Show file tree

Hide file tree

Showing 17 changed files with 622 additions and 263 deletions.
diff --git a/changelog.html b/changelog.html
@@ -70,6 +70,9 @@
   <li class=bug>
     Ensuring <code>/log/all</code> shows only <code>INFO</code> and above messages, even if custom loggers display <code>FINE</code> or below.
     (<a href="https://issues.jenkins-ci.org/browse/JENKINS-18959">issue 18959</a>)
+  <li class=rfe>
+    Added a new monitor that detects and fixse out-of-order builds records.
+    (<a href="https://issues.jenkins-ci.org/browse/JENKINS-18289">issue 18289</a>)
 </ul>
 </div><!--=TRUNK-END=-->
 

diff --git a/core/src/main/java/jenkins/diagnostics/ooom/BuildPtr.java b/core/src/main/java/jenkins/diagnostics/ooom/BuildPtr.java
@@ -1,21 +1,28 @@
 package jenkins.diagnostics.ooom;
 
+import hudson.FilePath;
+import hudson.model.Job;
+import hudson.model.TaskListener;
+
 import java.io.File;
+import java.io.IOException;
 import java.util.Comparator;
 
 /**
  * ID and build number of one build.
  */
-final class BuildPtr {
+public final class BuildPtr implements Comparable<BuildPtr> {
+    final Job job;
+
     final File buildDir;
     /**
      * Timestamp build ID.
      */
-    final String id;
+    public final String id;
     /**
      * Build number found from the disk.
      */
-    final int n;
+    public final int n;
 
     /**
      * Position of this build according to the ordering induced by {@link #n}
@@ -28,7 +35,8 @@ final class BuildPtr {
     int posByID;
 
 
-    BuildPtr(File buildDir, int n) {
+    BuildPtr(Job job, File buildDir, int n) {
+        this.job = job;
         this.n = n;
         this.id = buildDir.getName();
         this.buildDir = buildDir;
@@ -70,4 +78,31 @@ private static int signOfCompare(int a, int b) {
         if (a<b)    return -1;
         return 0;
     }
+
+    /**
+     * Fix the problem by moving the out of order builds into a place that Jenkins won't look at.
+     *
+     * TODO: another way to fix this is by adjusting the ID and pretend that the build happened
+     * at a different timestamp.
+     */
+    public void fix(TaskListener listener) throws IOException, InterruptedException {
+        File dir = new File(job.getRootDir(), "outOfOrderBuilds");
+        dir.mkdirs();
+        File dst = new File(dir, buildDir.getName());
+        listener.getLogger().println("Renaming "+dir);
+        listener.getLogger().println("  -> "+dst);
+        if (!buildDir.renameTo(dst)) {
+            FilePath bd = new FilePath(buildDir);
+            bd.copyRecursiveTo(new FilePath(dst));
+            bd.deleteRecursive();
+        }
+
+        // if there's a symlink delete it
+        new File(buildDir.getParentFile(),String.valueOf(n)).delete();
+    }
+
+    @Override
+    public int compareTo(BuildPtr that) {
+        return this.n - that.n;
+    }
 }
diff --git a/core/src/main/java/jenkins/diagnostics/ooom/OutOfOrderBuildDetector.java b/core/src/main/java/jenkins/diagnostics/ooom/OutOfOrderBuildDetector.java
@@ -22,8 +22,8 @@ public class OutOfOrderBuildDetector extends AsyncPeriodicWork {
     private OutOfOrderBuildMonitor monitor;
 
 
-    public OutOfOrderBuildDetector(String name) {
-        super(name);
+    public OutOfOrderBuildDetector() {
+        super("Out of order build detection");
     }
 
     @Override

diff --git a/core/src/main/java/jenkins/diagnostics/ooom/OutOfOrderBuildMonitor.java b/core/src/main/java/jenkins/diagnostics/ooom/OutOfOrderBuildMonitor.java
@@ -1,8 +1,14 @@
 package jenkins.diagnostics.ooom;
 
 import hudson.Extension;
-import hudson.model.AdministrativeMonitor;
+import hudson.model.TaskListener;
+import hudson.util.HttpResponses;
+import jenkins.management.AsynchronousAdministrativeMonitor;
+import org.kohsuke.stapler.HttpResponse;
+import org.kohsuke.stapler.interceptor.RequirePOST;
 
+import java.io.File;
+import java.util.Collections;
 import java.util.LinkedHashSet;
 import java.util.Set;
 
@@ -12,15 +18,53 @@
  * @author Kohsuke Kawaguchi
  */
 @Extension
-public class OutOfOrderBuildMonitor extends AdministrativeMonitor {
-    private final Set<Problem> problems = new LinkedHashSet<Problem>();
+public class OutOfOrderBuildMonitor extends AsynchronousAdministrativeMonitor {
+    private final Set<Problem> problems = Collections.synchronizedSet(new LinkedHashSet<Problem>());
 
     @Override
     public boolean isActivated() {
-        return !problems.isEmpty();
+        return !problems.isEmpty() || getLogFile().exists();
     }
 
     void addProblem(Problem p) {
         problems.add(p);
     }
+
+    public Set<Problem> getProblems() {
+        return Collections.unmodifiableSet(new LinkedHashSet<Problem>(problems));
+    }
+
+    @RequirePOST
+    public HttpResponse doFix() {
+        start(false);
+        return HttpResponses.forwardToPreviousPage();
+    }
+
+    /**
+     * Discards the current log file so that the "stuff is completed" message will be gone.
+     */
+    @RequirePOST
+    public HttpResponse doDismiss() {
+        getLogFile().delete();
+        return HttpResponses.forwardToPreviousPage();
+    }
+
+    @Override
+    public String getDisplayName() {
+        return "Fix Out-of-order Builds";
+    }
+
+    @Override
+    public File getLogFile() {
+        return super.getLogFile();
+    }
+
+    @Override
+    protected void fix(TaskListener listener) throws Exception {
+        Set<Problem> problems = getProblems();
+        for (Problem problem : problems) {
+            problem.fix(listener);
+        }
+        this.problems.removeAll(problems);
+    }
 }