Skip to content

Commit

Permalink
Merge pull request #103 from jglick/customMasterDir-JENKINS-38837
Browse files Browse the repository at this point in the history 
[JENKINS-38837] Handle custom master workspace directory
  • Loading branch information
@stephenc
stephenc committed Jun 15, 2017
2 parents bb1f94f + 5f615a9 commit 1610811
Show file tree
Hide file tree
Showing 2 changed files with 84 additions and 30 deletions.
72 changes: 47 additions & 25 deletions src/main/java/jenkins/branch/WorkspaceLocatorImpl.java
View file
Expand Up @@ -26,18 +26,22 @@

import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Charsets;
import com.google.common.collect.ImmutableMap;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.FilePath;
import hudson.Util;
import hudson.model.Computer;
import hudson.model.Item;
import hudson.model.Node;
import hudson.model.Slave;
import hudson.model.TopLevelItem;
import hudson.model.listeners.ItemListener;
import java.io.File;
import java.io.IOException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
Expand All @@ -58,9 +62,10 @@ public class WorkspaceLocatorImpl extends WorkspaceLocator {

private static final Logger LOGGER = Logger.getLogger(WorkspaceLocatorImpl.class.getName());

static final int PATH_MAX_DEFAULT = 80;
/** The most characters to allow in a workspace directory name, relative to the root. Zero to disable altogether. */
// TODO 2.4+ use SystemProperties
static /* not final */ int PATH_MAX = Integer.getInteger(WorkspaceLocatorImpl.class.getName() + ".PATH_MAX", 80);
static /* not final */ int PATH_MAX = Integer.getInteger(WorkspaceLocatorImpl.class.getName() + ".PATH_MAX", PATH_MAX_DEFAULT);

@Override
public FilePath locate(TopLevelItem item, Node node) {
Expand All @@ -72,7 +77,11 @@ public FilePath locate(TopLevelItem item, Node node) {
}
String minimized = minimize(item.getFullName());
if (node instanceof Jenkins) {
return ((Jenkins) node).getRootPath().child("workspace/" + minimized);
String workspaceDir = ((Jenkins) node).getRawWorkspaceDir();
if (!workspaceDir.contains("ITEM_FULL")) {
LOGGER.log(Level.WARNING, "JENKINS-34564 path sanitization ineffective when using legacy Workspace Root Directory ‘{0}’; switch to $'{'JENKINS_HOME'}'/workspace/$'{'ITEM_FULLNAME'}' as in JENKINS-8446 / JENKINS-21942", workspaceDir);
}
return new FilePath(new File(expandVariablesForDirectory(workspaceDir, minimized, item.getRootDir().getPath())));
} else if (node instanceof Slave) {
FilePath root = ((Slave) node).getWorkspaceRoot();
return root != null ? root.child(minimized) : null;
Expand All @@ -81,7 +90,17 @@ public FilePath locate(TopLevelItem item, Node node) {
}
}

static String uniqueSuffix(String name) {
/** copied from {@link Jenkins} */
static String expandVariablesForDirectory(String base, String itemFullName, String itemRootDir) {
return Util.replaceMacro(base, ImmutableMap.of(
"JENKINS_HOME", Jenkins.getActiveInstance().getRootDir().getPath(),
"ITEM_ROOTDIR", itemRootDir,
"ITEM_FULLNAME", itemFullName, // legacy, deprecated
"ITEM_FULL_NAME", itemFullName.replace(':','$'))); // safe, see JENKINS-12251

}

private static String uniqueSuffix(String name) {
// TODO still in beta: byte[] sha256 = Hashing.sha256().hashString(name).asBytes();
byte[] sha256;
try {
Expand Down Expand Up @@ -120,16 +139,18 @@ public static class Deleter extends ItemListener {

@Override
public void onDeleted(Item item) {
if (item.getParent() instanceof MultiBranchProject) {
final String suffix = uniqueSuffix(item.getFullName());
Jenkins jenkins = Jenkins.getActiveInstance();
Computer.threadPoolForRemoting.submit(new CleanupTask(suffix, jenkins.getRootPath().child("workspace"), "master"));
if (!(item instanceof TopLevelItem)) {
return;
}
TopLevelItem tli = (TopLevelItem) item;
Jenkins jenkins = Jenkins.getActiveInstance();
FilePath masterLoc = new WorkspaceLocatorImpl().locate(tli, jenkins);
if (masterLoc != null) {
Computer.threadPoolForRemoting.submit(new CleanupTask(masterLoc, "master"));
for (Node node : jenkins.getNodes()) {
if (node instanceof Slave) {
FilePath root = ((Slave) node).getWorkspaceRoot();
if (root != null) {
Computer.threadPoolForRemoting.submit(new CleanupTask(suffix, root, node.getNodeName()));
}
FilePath slaveLoc = new WorkspaceLocatorImpl().locate(tli, node);
if (slaveLoc != null) {
Computer.threadPoolForRemoting.submit(new CleanupTask(slaveLoc, node.getNodeName()));
}
}
}
Expand All @@ -156,41 +177,42 @@ private static synchronized void taskFinished() {

private static class CleanupTask implements Runnable {

/** @see #uniqueSuffix */
@NonNull
private final String suffix;

@NonNull
private final FilePath root;
private final FilePath loc;

@NonNull
private final String nodeName;

CleanupTask(String suffix, FilePath root, String nodeName) {
this.suffix = suffix;
this.root = root;
CleanupTask(FilePath loc, String nodeName) {
this.loc = loc;
this.nodeName = nodeName;
taskStarted();
}

@Override
public void run() {
String base = loc.getName();
FilePath parent = loc.getParent();
if (parent == null) { // unlikely but just in case
return;
}
Thread t = Thread.currentThread();
String oldName = t.getName();
t.setName(oldName + ": deleting workspace in " + suffix + " on " + nodeName);
t.setName(oldName + ": deleting workspace in " + loc + " on " + nodeName);
try {
try (Timeout timeout = Timeout.limit(5, TimeUnit.MINUTES)) {
if (!root.isDirectory()) {
List<FilePath> dirs = parent.listDirectories();
if (dirs == null) { // impossible as of https://github.com/jenkinsci/jenkins/pull/2914
return;
}
for (FilePath child : root.listDirectories()) {
if (child.getName().contains(suffix)) {
for (FilePath child : dirs) {
if (child.getName().startsWith(base)) {
LOGGER.log(Level.INFO, "deleting obsolete workspace {0} on {1}", new Object[] {child, nodeName});
child.deleteRecursive();
}
}
} catch (IOException | InterruptedException x) {
LOGGER.log(Level.WARNING, "could not clean up workspace directories under " + root + " on " + nodeName, x);
LOGGER.log(Level.WARNING, "could not clean up workspace directory " + loc + " on " + nodeName, x);
}
} finally {
t.setName(oldName);
Expand Down
42 changes: 37 additions & 5 deletions src/test/java/jenkins/branch/WorkspaceLocatorImplTest.java
View file
Expand Up @@ -24,17 +24,21 @@

package jenkins.branch;

import hudson.FilePath;
import hudson.model.FreeStyleProject;
import hudson.scm.NullSCM;
import hudson.slaves.DumbSlave;
import hudson.slaves.WorkspaceList;
import java.io.File;
import java.lang.reflect.Field;
import java.util.Collections;
import static jenkins.branch.NoTriggerBranchPropertyTest.showComputation;
import jenkins.branch.harness.MultiBranchImpl;
import jenkins.model.Jenkins;
import jenkins.scm.impl.SingleSCMSource;
import org.junit.Test;
import static org.junit.Assert.*;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Rule;
import org.jvnet.hudson.test.BuildWatcher;
Expand All @@ -49,10 +53,14 @@ public class WorkspaceLocatorImplTest {
@Rule
public JenkinsRule r = new JenkinsRule();

@Before
public void defaultPathMax() {
WorkspaceLocatorImpl.PATH_MAX = WorkspaceLocatorImpl.PATH_MAX_DEFAULT;
}

@WithoutJenkins
@Test
public void minimize() {
WorkspaceLocatorImpl.PATH_MAX = 80;
assertEquals("a_b-NX345YSMOYT4QUL4OO7V6EGKM57BBNSYVIXGXHCE4KAEVPV5KZYQ", WorkspaceLocatorImpl.minimize("a/b"));
assertEquals("a_b_c_d-UMWYJ45JQ6FA3WXMSI3YEOLVQ5P6SFYWN26FRECRSFBUGUD27Y5A", WorkspaceLocatorImpl.minimize("a/b/c/d"));
assertEquals("stuff_dev_flow-L5GKER67QGVMJ2UD3JCSGKEV2ACON2O4VO4RNUZ27HGUY32SYVXQ", WorkspaceLocatorImpl.minimize("stuff/dev%2Fflow"));
Expand Down Expand Up @@ -111,14 +119,12 @@ public void minimize() {
assertEquals("W-LRVIZHY37B", WorkspaceLocatorImpl.minimize("blahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahXYZW"));
assertEquals("V-KLYOGWEJOD", WorkspaceLocatorImpl.minimize("blahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahXYZWV"));
assertEquals("U-OSF24EPB4C", WorkspaceLocatorImpl.minimize("blahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahblahXYZWVU"));

// Reset back to 80 for other tests that assume it is 80.
WorkspaceLocatorImpl.PATH_MAX = 80;
}

@Issue("JENKINS-34564")
@Issue({"JENKINS-34564", "JENKINS-38837"})
@Test
public void locate() throws Exception {
assertEquals("${JENKINS_HOME}/workspace/${ITEM_FULLNAME}", r.jenkins.getRawWorkspaceDir());
MultiBranchImpl stuff = r.createProject(MultiBranchImpl.class, "stuff");
stuff.getSourcesList().add(new BranchSource(new SingleSCMSource(null, "dev/flow", new NullSCM())));
stuff.scheduleBuild2(0).getFuture().get();
Expand All @@ -131,6 +137,15 @@ public void locate() throws Exception {
assertEquals(slave.getWorkspaceRoot().child("stuff_dev_flow-L5GKER67QGVMJ2UD3JCSGKEV2ACON2O4VO4RNUZ27HGUY32SYVXQ"), slave.getWorkspaceFor(master));
FreeStyleProject unrelated = r.createFreeStyleProject("100% crazy");
assertEquals(r.jenkins.getRootPath().child("workspace/100% crazy"), r.jenkins.getWorkspaceFor(unrelated));
// Checking other values of workspaceDir.
Field workspaceDir = Jenkins.class.getDeclaredField("workspaceDir"); // currently settable only by Jenkins.doConfigSubmit
workspaceDir.setAccessible(true);
// Poor historical default, and as per JENKINS-21942 even possible after some startup scenarios:
workspaceDir.set(r.jenkins, "${ITEM_ROOTDIR}/workspace");
assertEquals("JENKINS-34564 inactive in this case", new FilePath(master.getRootDir()).child("workspace"), r.jenkins.getWorkspaceFor(master));
// JENKINS-38837: customized root.
workspaceDir.set(r.jenkins, "${JENKINS_HOME}/ws/${ITEM_FULLNAME}");
assertEquals("ITEM_FULLNAME interpreted a little differently", r.jenkins.getRootPath().child("ws/stuff_dev_flow-L5GKER67QGVMJ2UD3JCSGKEV2ACON2O4VO4RNUZ27HGUY32SYVXQ"), r.jenkins.getWorkspaceFor(master));
}

@Issue({"JENKINS-2111", "JENKINS-41068"})
Expand Down Expand Up @@ -173,6 +188,23 @@ public void delete() throws Exception {
assertEquals(Collections.singletonList(r.jenkins.getRootPath().child("workspace/p_master-NFABYX74Y6QHVCY2OKHXKUN4SSHQIWYYSJW7JE3FM65W5M5OSXMA")), r.jenkins.getRootPath().child("workspace").listDirectories());
assertEquals(Collections.singletonList(slave.getWorkspaceRoot().child("p_master-NFABYX74Y6QHVCY2OKHXKUN4SSHQIWYYSJW7JE3FM65W5M5OSXMA")), slave.getWorkspaceRoot().listDirectories());
assertFalse(pr1Root.isDirectory());
// Also check behavior with customized paths.
WorkspaceLocatorImpl.PATH_MAX = 40;
p.getSourcesList().add(pr1Source);
p.scheduleBuild2(0).getFuture().get();
r.waitUntilNoActivity();
showComputation(p);
pr1 = r.jenkins.getItemByFullName("p/PR-1", FreeStyleProject.class);
assertNotNull(pr1);
pr1.setAssignedNode(slave);
assertEquals(slave.getWorkspaceRoot().child("p_PR-1-4FMSDR3M7ZFZIJ2EAYSJ4FQ5N"), slave.getWorkspaceFor(pr1));
assertEquals(2, r.buildAndAssertSuccess(pr1).getNumber());
p.getSourcesList().remove(pr1Source);
p.scheduleBuild2(0).getFuture().get();
r.waitUntilNoActivity();
showComputation(p);
WorkspaceLocatorImpl.Deleter.waitForTasksToFinish();
assertEquals(Collections.singletonList(slave.getWorkspaceRoot().child("p_master-NFABYX74Y6QHVCY2OKHXKUN4SSHQIWYYSJW7JE3FM65W5M5OSXMA")), slave.getWorkspaceRoot().listDirectories());
}

}

0 comments on commit 1610811

Please sign in to comment.