Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[FIXED JENKINS-44784] Ensure workspace cleanup for deleted branch pro…
…jects proceeds asynchronously, and will time out if the agent is stuck.
  • Loading branch information
jglick committed Jun 8, 2017
1 parent 4706218 commit bc1fe57
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 14 deletions.
62 changes: 62 additions & 0 deletions src/main/java/jenkins/branch/Timeout.java
@@ -0,0 +1,62 @@
/*
* The MIT License
*
* Copyright 2017 CloudBees, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/

package jenkins.branch;

import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import jenkins.util.Timer;

// TODO copied from org.jenkinsci.plugins.workflow.support.concurrent pending JENKINS-44785
class Timeout implements AutoCloseable {

private static final Logger LOGGER = Logger.getLogger(Timeout.class.getName());

private final ScheduledFuture<?> task;

private Timeout(ScheduledFuture<?> task) {
this.task = task;
}

@Override public void close() {
task.cancel(false);
}

public static Timeout limit(final long time, final TimeUnit unit) {
final Thread thread = Thread.currentThread();
return new Timeout(Timer.get().schedule(new Runnable() {
@Override public void run() {
if (LOGGER.isLoggable(Level.FINE)) {
Throwable t = new Throwable();
t.setStackTrace(thread.getStackTrace());
LOGGER.log(Level.FINE, "Interrupting " + thread + " after " + time + " " + unit, t);
}
thread.interrupt();
}
}, time, unit));
}

}
84 changes: 70 additions & 14 deletions src/main/java/jenkins/branch/WorkspaceLocatorImpl.java
Expand Up @@ -24,9 +24,12 @@

package jenkins.branch;

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.FilePath;
import hudson.model.Computer;
import hudson.model.Item;
import hudson.model.Node;
import hudson.model.Slave;
Expand All @@ -35,6 +38,7 @@
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import jenkins.model.Jenkins;
Expand Down Expand Up @@ -117,31 +121,83 @@ public static class Deleter extends ItemListener {
@Override
public void onDeleted(Item item) {
if (item.getParent() instanceof MultiBranchProject) {
String suffix = uniqueSuffix(item.getFullName());
final String suffix = uniqueSuffix(item.getFullName());
Jenkins jenkins = Jenkins.getActiveInstance();
cleanUp(suffix, jenkins.getRootPath().child("workspace"), jenkins);
Computer.threadPoolForRemoting.submit(new CleanupTask(suffix, jenkins.getRootPath().child("workspace"), "master"));
for (Node node : jenkins.getNodes()) {
if (node instanceof Slave) {
cleanUp(suffix, ((Slave) node).getWorkspaceRoot(), node);
FilePath root = ((Slave) node).getWorkspaceRoot();
if (root != null) {
Computer.threadPoolForRemoting.submit(new CleanupTask(suffix, root, node.getNodeName()));
}
}
}
}
}

private void cleanUp(String suffix, FilePath root, Node node) {
try {
if (root == null || !root.isDirectory()) {
return;
}
for (FilePath child : root.listDirectories()) {
if (child.getName().contains(suffix)) {
LOGGER.log(Level.INFO, "deleting obsolete workspace {0} on {1}", new Object[] {child, node.getNodeName()});
child.deleteRecursive();
/** Number of {@link CleanupTask} which have been scheduled but not yet completed. */
private static int runningTasks;

@VisibleForTesting
static synchronized void waitForTasksToFinish() throws InterruptedException {
while (runningTasks > 0) {
Deleter.class.wait();
}
}

private static synchronized void taskStarted() {
runningTasks++;
}

private static synchronized void taskFinished() {
runningTasks--;
Deleter.class.notifyAll();
}

private static class CleanupTask implements Runnable {

/** @see #uniqueSuffix */
@NonNull
private final String suffix;

@NonNull
private final FilePath root;

@NonNull
private final String nodeName;

CleanupTask(String suffix, FilePath root, String nodeName) {
this.suffix = suffix;
this.root = root;
this.nodeName = nodeName;
taskStarted();
}

@Override
public void run() {
Thread t = Thread.currentThread();
String oldName = t.getName();
t.setName(oldName + ": deleting workspace in " + suffix + " on " + nodeName);
try {
try (Timeout timeout = Timeout.limit(5, TimeUnit.MINUTES)) {
if (!root.isDirectory()) {
return;
}
for (FilePath child : root.listDirectories()) {
if (child.getName().contains(suffix)) {
LOGGER.log(Level.INFO, "deleting obsolete workspace {0} on {1}", new Object[] {child, nodeName});
child.deleteRecursive();
}
}
} catch (IOException | InterruptedException x) {
LOGGER.log(Level.WARNING, "could not clean up workspace directories under " + root + " on " + nodeName, x);
}
} finally {
t.setName(oldName);
taskFinished();
}
} catch (IOException | InterruptedException x) {
LOGGER.log(Level.WARNING, "could not clean up workspace directories under " + root + " on " + node.getNodeName(), x);
}

}

}
Expand Down
1 change: 1 addition & 0 deletions src/test/java/jenkins/branch/WorkspaceLocatorImplTest.java
Expand Up @@ -168,6 +168,7 @@ public void delete() throws Exception {
p.scheduleBuild2(0).getFuture().get();
r.waitUntilNoActivity();
showComputation(p);
WorkspaceLocatorImpl.Deleter.waitForTasksToFinish();
assertEquals(Collections.singletonList(master), r.jenkins.getAllItems(FreeStyleProject.class));
assertEquals(Collections.singletonList(r.jenkins.getRootPath().child("workspace/p_master-NFABYX74Y6QHVCY2OKHXKUN4SSHQIWYYSJW7JE3FM65W5M5OSXMA")), r.jenkins.getRootPath().child("workspace").listDirectories());
assertEquals(Collections.singletonList(slave.getWorkspaceRoot().child("p_master-NFABYX74Y6QHVCY2OKHXKUN4SSHQIWYYSJW7JE3FM65W5M5OSXMA")), slave.getWorkspaceRoot().listDirectories());
Expand Down

0 comments on commit bc1fe57

Please sign in to comment.