Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[JENKINS-49205] Allow file transfers without Remoting-based CLI.
  • Loading branch information
jglick committed Jan 26, 2018
1 parent e5e2173 commit e3438da
Show file tree
Hide file tree
Showing 3 changed files with 78 additions and 19 deletions.
1 change: 1 addition & 0 deletions pom.xml
Expand Up @@ -23,6 +23,7 @@
<properties>
<jenkins.version>2.60.3</jenkins.version>
<java.level>8</java.level>
<jenkins-test-harness.version>2.34-20180126.205758-1</jenkins-test-harness.version> <!-- TODO https://github.com/jenkinsci/jenkins-test-harness/pull/94 -->
</properties>

<scm>
Expand Down
34 changes: 19 additions & 15 deletions src/main/java/hudson/plugins/distfork/DistForkCommand.java
Expand Up @@ -60,27 +60,29 @@ public class DistForkCommand extends CLICommand {
public List<String> commands = new ArrayList<String>();

@Option(name="-z",metaVar="FILE",
usage="Zip/tgz file to be extracted into the target remote machine before execution of the command")
usage="Zip/tgz file to be extracted into the target remote machine before execution of the command; " +
"requires -remoting unless you pass =zip or =tgz in which case stdin is used")
public String zip;

@Option(name="-Z",metaVar="FILE",
usage="Bring back the newly added/updated files in the target remote machine after the end of the command " +
"by creating a zip/tgz bundle and place this in the local file system by this name.")
"by creating a zip/tgz bundle and place this in the local file system by this name; " +
"requires -remoting unless you pass =zip or =tgz in which case stdout is used")
public String returnZip;

@Option(name="-e",usage="Environment variables to set to the launched process",metaVar="NAME=VAL")
public Map<String,String> envs = new HashMap<String,String>();

@Option(name="-f",usage="Local files to be copied to remote locations before the execution of a task",metaVar="REMOTE=LOCAL")
@Option(name="-f",usage="Local files to be copied to remote locations before the execution of a task; requires -remoting",metaVar="REMOTE=LOCAL")
public Map<String,String> files = new HashMap<String,String>();

@Option(name="-F",usage="Remote files to be copied back to local locations after the execution of a task",metaVar="LOCAL=REMOTE")
@Option(name="-F",usage="Remote files to be copied back to local locations after the execution of a task; requires -remoting",metaVar="LOCAL=REMOTE")
public Map<String,String> returnFiles = new HashMap<String,String>();

@Option(name="-L",usage="Local to remote port forwarding",handler=PortForwardingArgumentHandler.class)
@Option(name="-L",usage="Local to remote port forwarding; requires -remoting",handler=PortForwardingArgumentHandler.class)
public List<PortSpec> l2rFowrarding = new ArrayList<PortSpec>();

@Option(name="-R",usage="Remote to local port forwarding",handler=PortForwardingArgumentHandler.class)
@Option(name="-R",usage="Remote to local port forwarding; requires -remoting",handler=PortForwardingArgumentHandler.class)
public List<PortSpec> r2lFowrarding = new ArrayList<PortSpec>();

public String getShortDescription() {
Expand Down Expand Up @@ -177,8 +179,8 @@ public void run() {

{// copy over files
if(zip!=null) {
BufferedInputStream in = new BufferedInputStream(new FilePath(checkChannel(), zip).read());
if(zip.endsWith(".zip"))
BufferedInputStream in = new BufferedInputStream(zip.matches("=(zip|tgz)") ? stdin : new FilePath(checkChannel(), zip).read());
if(zip.endsWith("zip"))
workDir.unzipFrom(in);
else
workDir.untarFrom(in, TarCompression.GZIP);
Expand All @@ -190,8 +192,12 @@ public void run() {
}

List<Closeable> cleanUpList = new ArrayList<Closeable>();
setUpPortForwarding(l2rFowrarding, checkChannel(), c.getChannel(), cleanUpList);
setUpPortForwarding(r2lFowrarding, c.getChannel(), checkChannel(), cleanUpList);
if (!l2rFowrarding.isEmpty()) {
setUpPortForwarding(l2rFowrarding, checkChannel(), c.getChannel(), cleanUpList);
}
if (!r2lFowrarding.isEmpty()) {
setUpPortForwarding(r2lFowrarding, c.getChannel(), checkChannel(), cleanUpList);
}

try {
long startTime = c.getChannel().call(new GetSystemTime());
Expand All @@ -211,16 +217,14 @@ public void run() {
}

if (returnZip!=null) {
OutputStream os = new BufferedOutputStream(new FilePath(checkChannel(), returnZip).write());
try {
try (OutputStream os = new BufferedOutputStream(returnZip.matches("=(zip|tgz)") ? stdout : new FilePath(checkChannel(), returnZip).write())) {
RootCutOffFilter scanner = new RootCutOffFilter(new TimestampFilter(startTime));
if(returnZip.endsWith(".zip")) {
if(returnZip.endsWith("zip")) {
workDir.zip(os,scanner);
} else {
workDir.tar(TarCompression.GZIP.compress(os),scanner);
}
} finally {
os.close();
os.flush();
}
}
}
Expand Down
62 changes: 58 additions & 4 deletions src/test/java/hudson/plugins/distfork/DistForkCommandTest.java
Expand Up @@ -49,9 +49,14 @@
import hudson.slaves.DumbSlave;
import java.io.File;
import hudson.util.StreamTaskListener;
import java.io.ByteArrayInputStream;
import java.util.logging.Level;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import jenkins.model.Jenkins;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.jenkinsci.test.acceptance.docker.fixtures.JavaContainer;
import static org.hamcrest.Matchers.*;
import org.jenkinsci.test.acceptance.docker.DockerRule;
Expand Down Expand Up @@ -262,16 +267,65 @@ public void plainCLINamedFileTransfers() throws Exception {
File b = tmp.newFile();
FileUtils.write(b, "world");
File c = tmp.newFile();
CLICommandInvoker.Result r = new CLICommandInvoker(jr, new DistForkCommand()).invokeWithArgs(
"dist-fork", "-l", "docker", "-f", "/home/test/a=" + a, "-f", "/home/test/b=" + b, "-F", c + "=/home/test/c", "sh", "-c", "cat /home/test/a /home/test/b > /home/test/c");
CLICommandInvoker.Result r = new CLICommandInvoker(jr, new DistForkCommand()).
invokeWithArgs("-l", "docker", "-f", "/home/test/a=" + a, "-f", "/home/test/b=" + b, "-F", c + "=/home/test/c", "sh", "-c", "cat /home/test/a /home/test/b > /home/test/c");
assertThat(r, CLICommandInvoker.Matcher.failedWith(-1));
assertThat(r.toString(), r.stderr(), containsString("https://jenkins.io/redirect/cli-command-requires-channel"));
}

@Issue("JENKINS-49205")
@Test
public void plainCLIStdinFileTransfers() throws Exception {
// TODO but when using = it should be possible
public void plainCLIStdinFileTransfersMaster() throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
ZipEntry ze = new ZipEntry("a");
zos.putNextEntry(ze);
zos.write("hello ".getBytes());
zos.closeEntry();
ze = new ZipEntry("b");
zos.putNextEntry(ze);
zos.write("world".getBytes());
zos.closeEntry();
}
CLICommandInvoker.Result r = new CLICommandInvoker(jr, new DistForkCommand()).
withStdin(new ByteArrayInputStream(baos.toByteArray())).
invokeWithArgs("-z", "=zip", "-Z", "=zip", "sh", "-c", "sleep 1; cat a b > c; rm a b");
assertThat(r, CLICommandInvoker.Matcher.succeeded());
try (ByteArrayInputStream bais = new ByteArrayInputStream(r.stdoutBinary()); ZipInputStream zis = new ZipInputStream(bais)) {
ZipEntry ze = zis.getNextEntry();
assertNotNull(ze);
assertEquals("c", ze.getName());
assertEquals("hello world", IOUtils.toString(zis));
assertNull(zis.getNextEntry());
}
}

@Issue("JENKINS-49205")
@Test
public void plainCLIStdinFileTransfersSlave() throws Exception {
registerSlave();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
ZipEntry ze = new ZipEntry("a");
zos.putNextEntry(ze);
zos.write("hello ".getBytes());
zos.closeEntry();
ze = new ZipEntry("b");
zos.putNextEntry(ze);
zos.write("world".getBytes());
zos.closeEntry();
}
CLICommandInvoker.Result r = new CLICommandInvoker(jr, new DistForkCommand()).
withStdin(new ByteArrayInputStream(baos.toByteArray())).
invokeWithArgs("-l", "docker", "-z", "=zip", "-Z", "=zip", "sh", "-c", "sleep 1; cat a b > c; rm a b");
assertThat(r, CLICommandInvoker.Matcher.succeeded());
try (ByteArrayInputStream bais = new ByteArrayInputStream(r.stdoutBinary()); ZipInputStream zis = new ZipInputStream(bais)) {
ZipEntry ze = zis.getNextEntry();
assertNotNull(ze);
assertEquals("c", ze.getName());
assertEquals("hello world", IOUtils.toString(zis));
assertNull(zis.getNextEntry());
}
}

}

0 comments on commit e3438da

Please sign in to comment.