Skip to content

Commit

Permalink
Merge pull request #9 from jglick/plain-CLI-JENKINS-49205
Browse files Browse the repository at this point in the history 
 [JENKINS-49205] Allow file transfers without Remoting-based CLI
  • Loading branch information
@jglick
jglick committed Jan 30, 2018
2 parents 22e1d1b + 9a5e34c commit 277e7bb
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 27 deletions.
2 changes: 1 addition & 1 deletion Jenkinsfile
View file
@@ -1 +1 @@
buildPlugin()
buildPlugin(platforms: ['docker', 'windows'])
13 changes: 13 additions & 0 deletions pom.xml
View file
Expand Up @@ -23,6 +23,7 @@
<properties>
<jenkins.version>2.60.3</jenkins.version>
<java.level>8</java.level>
<jenkins-test-harness.version>2.34</jenkins-test-harness.version>
</properties>

<scm>
Expand All @@ -45,6 +46,18 @@
<version>1.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.test</groupId>
<artifactId>docker-fixtures</artifactId>
<version>1.5</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>ssh-slaves</artifactId>
<version>1.25</version>
<scope>test</scope>
</dependency>
</dependencies>

<repositories>
Expand Down
57 changes: 35 additions & 22 deletions src/main/java/hudson/plugins/distfork/DistForkCommand.java
View file
Expand Up @@ -3,6 +3,7 @@
import hudson.Extension;
import hudson.FilePath;
import hudson.FilePath.TarCompression;
import hudson.Functions;
import hudson.Launcher;
import hudson.Util;
import hudson.cli.CLICommand;
Expand Down Expand Up @@ -31,7 +32,6 @@
import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
Expand Down Expand Up @@ -60,27 +60,29 @@ public class DistForkCommand extends CLICommand {
public List<String> commands = new ArrayList<String>();

@Option(name="-z",metaVar="FILE",
usage="Zip/tgz file to be extracted into the target remote machine before execution of the command")
usage="Zip/tgz file to be extracted into the target remote machine before execution of the command; " +
"requires -remoting unless you pass =zip or =tgz in which case stdin is used")
public String zip;

@Option(name="-Z",metaVar="FILE",
usage="Bring back the newly added/updated files in the target remote machine after the end of the command " +
"by creating a zip/tgz bundle and place this in the local file system by this name.")
"by creating a zip/tgz bundle and place this in the local file system by this name; " +
"requires -remoting unless you pass =zip or =tgz in which case stdout is used")
public String returnZip;

@Option(name="-e",usage="Environment variables to set to the launched process",metaVar="NAME=VAL")
public Map<String,String> envs = new HashMap<String,String>();

@Option(name="-f",usage="Local files to be copied to remote locations before the execution of a task",metaVar="REMOTE=LOCAL")
@Option(name="-f",usage="Local files to be copied to remote locations before the execution of a task; requires -remoting",metaVar="REMOTE=LOCAL")
public Map<String,String> files = new HashMap<String,String>();

@Option(name="-F",usage="Remote files to be copied back to local locations after the execution of a task",metaVar="LOCAL=REMOTE")
@Option(name="-F",usage="Remote files to be copied back to local locations after the execution of a task; requires -remoting",metaVar="LOCAL=REMOTE")
public Map<String,String> returnFiles = new HashMap<String,String>();

@Option(name="-L",usage="Local to remote port forwarding",handler=PortForwardingArgumentHandler.class)
@Option(name="-L",usage="Local to remote port forwarding; requires -remoting",handler=PortForwardingArgumentHandler.class)
public List<PortSpec> l2rFowrarding = new ArrayList<PortSpec>();

@Option(name="-R",usage="Remote to local port forwarding",handler=PortForwardingArgumentHandler.class)
@Option(name="-R",usage="Remote to local port forwarding; requires -remoting",handler=PortForwardingArgumentHandler.class)
public List<PortSpec> r2lFowrarding = new ArrayList<PortSpec>();

public String getShortDescription() {
Expand Down Expand Up @@ -149,8 +151,16 @@ protected int run() throws Exception {
final int[] exitCode = new int[]{-1};

DistForkTask t = new DistForkTask(l, name, duration, new Runnable() {
@SuppressWarnings("deprecation") // checkChannel only used in -remoting modes
@Override
public void run() {
StreamTaskListener listener = new StreamTaskListener(stdout, Charset.defaultCharset());
StreamTaskListener listener;
try {
listener = new StreamTaskListener(stderr, getClientCharset());
} catch (IOException | InterruptedException x) {
Functions.printStackTrace(x, stderr);
return;
}
try {
Computer c = Computer.currentComputer();
Node n = c.getNode();
Expand All @@ -169,20 +179,25 @@ public void run() {

{// copy over files
if(zip!=null) {
BufferedInputStream in = new BufferedInputStream(new FilePath(channel, zip).read());
if(zip.endsWith(".zip"))
BufferedInputStream in = new BufferedInputStream(zip.matches("=(zip|tgz)") ? stdin : new FilePath(checkChannel(), zip).read());
if(zip.endsWith("zip"))
workDir.unzipFrom(in);
else
workDir.untarFrom(in, TarCompression.GZIP);
}

for (Entry<String, String> e : files.entrySet())
new FilePath(channel,e.getValue()).copyToWithPermission(workDir.child(e.getKey()));
for (Entry<String, String> e : files.entrySet()) {
new FilePath(checkChannel(), e.getValue()).copyToWithPermission(workDir.child(e.getKey()));
}
}

List<Closeable> cleanUpList = new ArrayList<Closeable>();
setUpPortForwarding(l2rFowrarding,channel,c.getChannel(),cleanUpList);
setUpPortForwarding(r2lFowrarding,c.getChannel(),channel,cleanUpList);
if (!l2rFowrarding.isEmpty()) {
setUpPortForwarding(l2rFowrarding, checkChannel(), c.getChannel(), cleanUpList);
}
if (!r2lFowrarding.isEmpty()) {
setUpPortForwarding(r2lFowrarding, c.getChannel(), checkChannel(), cleanUpList);
}

try {
long startTime = c.getChannel().call(new GetSystemTime());
Expand All @@ -193,25 +208,23 @@ public void run() {
if (!returnFiles.isEmpty() || returnZip!=null) {
stderr.println("Copying back files");
for (Entry<String, String> e : returnFiles.entrySet()) {
FilePath tmp = new FilePath(channel, e.getKey() + ".tmp");
FilePath actual = new FilePath(channel, e.getKey());
FilePath tmp = new FilePath(checkChannel(), e.getKey() + ".tmp");
FilePath actual = new FilePath(checkChannel(), e.getKey());
workDir.child(e.getValue()).copyToWithPermission(tmp);
if (actual.exists())
actual.delete();
tmp.renameTo(actual);
}

if (returnZip!=null) {
OutputStream os = new BufferedOutputStream(new FilePath(channel,returnZip).write());
try {
try (OutputStream os = new BufferedOutputStream(returnZip.matches("=(zip|tgz)") ? stdout : new FilePath(checkChannel(), returnZip).write())) {
RootCutOffFilter scanner = new RootCutOffFilter(new TimestampFilter(startTime));
if(returnZip.endsWith(".zip")) {
if(returnZip.endsWith("zip")) {
workDir.zip(os,scanner);
} else {
workDir.tar(TarCompression.GZIP.compress(os),scanner);
}
} finally {
os.close();
os.flush();
}
}
}
Expand All @@ -224,7 +237,7 @@ public void run() {
listener.error("Aborted");
exitCode[0] = -1;
} catch (Exception e) {
e.printStackTrace(listener.error("Failed to execute a process"));
Functions.printStackTrace(e, listener.error("Failed to execute a process"));
exitCode[0] = -1;
}
}
Expand Down
119 changes: 115 additions & 4 deletions src/test/java/hudson/plugins/distfork/DistForkCommandTest.java
View file
Expand Up @@ -23,6 +23,7 @@
*/
package hudson.plugins.distfork;

import hudson.Functions;
import hudson.Launcher;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
Expand All @@ -35,24 +36,36 @@
import org.jvnet.hudson.test.JenkinsRule;

import hudson.cli.CLI;
import hudson.cli.CLICommandInvoker;
import hudson.model.Computer;
import hudson.model.Item;
import hudson.model.Node.Mode;
import hudson.model.Queue;
import hudson.model.User;
import hudson.plugins.sshslaves.SSHLauncher;
import hudson.security.AccessDeniedException2;
import hudson.security.GlobalMatrixAuthorizationStrategy;
import hudson.security.HudsonPrivateSecurityRealm;
import hudson.slaves.Cloud;
import hudson.slaves.DumbSlave;
import java.io.File;
import hudson.util.StreamTaskListener;
import java.io.ByteArrayInputStream;
import java.util.logging.Level;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import jenkins.model.Jenkins;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.jenkinsci.test.acceptance.docker.fixtures.JavaContainer;
import static org.hamcrest.Matchers.*;
import org.jenkinsci.test.acceptance.docker.DockerRule;
import static org.junit.Assert.*;
import static org.junit.Assume.*;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.rules.TemporaryFolder;
import org.jvnet.hudson.test.LoggerRule;

public class DistForkCommandTest {
Expand All @@ -63,6 +76,13 @@ public class DistForkCommandTest {
@Rule
public LoggerRule logging = new LoggerRule();

@Rule
public TemporaryFolder tmp = new TemporaryFolder();

// could use DockerClassRule only if moved to another test suite
@Rule
public DockerRule<JavaContainer> docker = new DockerRule<>(JavaContainer.class);

/** JENKINS_24752: otherwise {@link #testUserWithBuildAccessOnCloud} waits a long time */
@BeforeClass
public static void runFaster() {
Expand Down Expand Up @@ -210,14 +230,105 @@ public void testUserWithBuildAccessOnCloud() throws Exception {
assertThat(result, allOf( containsString("Executing on mock-"), containsString(whoIAM)));
}

@SuppressWarnings("deprecation") // deliberately testing -remoting
private String commandAndOutput(String... args) throws Exception {
CLI cli = new CLI(jr.getURL());
try {
try (CLI cli = new CLI(jr.getURL())) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
cli.execute(Arrays.asList(args), new NullInputStream(0), baos, baos);
System.err.println(Arrays.toString(args) + " → " + baos);
return baos.toString();
} finally {
cli.close();
}
}

private void registerSlave() throws Exception {
JavaContainer c = docker.get();
DumbSlave s = new DumbSlave("docker", "/home/test/slave", new SSHLauncher(c.ipBound(22), c.port(22), "test", "test", "", ""));
jr.jenkins.addNode(s);
jr.waitOnline(s);
}

@Test
public void remotingCLINamedFileTransfers() throws Exception {
registerSlave();
File a = tmp.newFile();
FileUtils.write(a, "hello ");
File b = tmp.newFile();
FileUtils.write(b, "world");
File c = tmp.newFile();
String result = commandAndOutput("dist-fork", "-l", "docker", "-f", "/home/test/a=" + a, "-f", "/home/test/b=" + b, "-F", c + "=/home/test/c", "sh", "-c", "cat /home/test/a /home/test/b > /home/test/c");
assertThat(result, containsString("Executing on docker"));
assertEquals("hello world", FileUtils.readFileToString(c));
}

@Test
public void plainCLINamedFileTransfers() throws Exception {
registerSlave();
File a = tmp.newFile();
FileUtils.write(a, "hello ");
File b = tmp.newFile();
FileUtils.write(b, "world");
File c = tmp.newFile();
CLICommandInvoker.Result r = new CLICommandInvoker(jr, new DistForkCommand()).
invokeWithArgs("-l", "docker", "-f", "/home/test/a=" + a, "-f", "/home/test/b=" + b, "-F", c + "=/home/test/c", "sh", "-c", "cat /home/test/a /home/test/b > /home/test/c");
assertThat(r, CLICommandInvoker.Matcher.failedWith(-1));
assertThat(r.toString(), r.stderr(), containsString("https://jenkins.io/redirect/cli-command-requires-channel"));
}

@Issue("JENKINS-49205")
@Test
public void plainCLIStdinFileTransfersMaster() throws Exception {
assumeFalse("TODO would need to write an equivalent batch script", Functions.isWindows());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
ZipEntry ze = new ZipEntry("a");
zos.putNextEntry(ze);
zos.write("hello ".getBytes());
zos.closeEntry();
ze = new ZipEntry("b");
zos.putNextEntry(ze);
zos.write("world".getBytes());
zos.closeEntry();
}
CLICommandInvoker.Result r = new CLICommandInvoker(jr, new DistForkCommand()).
withStdin(new ByteArrayInputStream(baos.toByteArray())).
// TODO sleep necessary because TimestampFilter otherwise excludes files created immediately at start of process
invokeWithArgs("-z", "=zip", "-Z", "=zip", "sh", "-c", "sleep 1; cat a b > c; rm a b");
assertThat(r, CLICommandInvoker.Matcher.succeeded());
try (ByteArrayInputStream bais = new ByteArrayInputStream(r.stdoutBinary()); ZipInputStream zis = new ZipInputStream(bais)) {
ZipEntry ze = zis.getNextEntry();
assertNotNull(ze);
assertEquals("c", ze.getName());
assertEquals("hello world", IOUtils.toString(zis));
assertNull(zis.getNextEntry());
}
}

@Issue("JENKINS-49205")
@Test
public void plainCLIStdinFileTransfersSlave() throws Exception {
registerSlave();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
ZipEntry ze = new ZipEntry("a");
zos.putNextEntry(ze);
zos.write("hello ".getBytes());
zos.closeEntry();
ze = new ZipEntry("b");
zos.putNextEntry(ze);
zos.write("world".getBytes());
zos.closeEntry();
}
CLICommandInvoker.Result r = new CLICommandInvoker(jr, new DistForkCommand()).
withStdin(new ByteArrayInputStream(baos.toByteArray())).
invokeWithArgs("-l", "docker", "-z", "=zip", "-Z", "=zip", "sh", "-c", "sleep 1; cat a b > c; rm a b");
assertThat(r, CLICommandInvoker.Matcher.succeeded());
try (ByteArrayInputStream bais = new ByteArrayInputStream(r.stdoutBinary()); ZipInputStream zis = new ZipInputStream(bais)) {
ZipEntry ze = zis.getNextEntry();
assertNotNull(ze);
assertEquals("c", ze.getName());
assertEquals("hello world", IOUtils.toString(zis));
assertNull(zis.getNextEntry());
}
}

}

0 comments on commit 277e7bb

Please sign in to comment.