Merge pull request #40 from jenkinsci/script-from-scm-JENKINS-26101

[FIXED JENKINS-26101] Load script from SCM
jenkinsci · Jan 24, 2015 · f1e8e16 · f1e8e16
2 parents e2947e9 + 3bbd33f
commit f1e8e16
Show file tree

Hide file tree

Showing 34 changed files with 679 additions and 288 deletions.
diff --git a/CHANGES.md b/CHANGES.md
@@ -5,6 +5,7 @@ Only noting significant user-visible or major API changes, not internal code cle
 ## 1.2 (upcoming)
 
 ### User changes
+* JENKINS-26101: the complete workflow script can now be loaded from an SCM repository of your choice.
 * JENKINS-26149: the `build` step did not survive Jenkins restarts while running.
 * JENKINS-25570: added `waitUntil` step.
 * JENKINS-25924: added `error` step.

diff --git a/TUTORIAL.md b/TUTORIAL.md
@@ -593,10 +593,19 @@ Consult the [Docker demo](demo/README.md) for an example of a flow using multipl
 Complex flows would be cumbersome to write and maintain in the textarea provided in the Jenkins job configuration.
 Therefore it makes sense to load the program from another source, one that you can maintain using version control and standalone Groovy editors.
 
+## Entire script from SCM
+
+The easiest way to do this is to select _Groovy CPS DSL from SCM_ when defining the workflow.
+In that case you do not enter any Groovy code in the Jenkins UI; you just indicate where in source code you want to retrieve the program.
+(If you update this repository, a new build will be triggered, so long as your job is configured with an SCM polling trigger.)
+
+## Manual loading
+
+For some cases you may prefer to explicitly load Groovy script text from some source.
 The standard Groovy `evaluate` function can be used, but most likely you will want to load a flow definition from a workspace.
 For this purpose you can use the `load` step, which takes a filename in the workspace and runs it as Groovy source text.
 The loaded file can either contain statements at top level, which are run immediately; or it can define functions and return `this`, in which case the result of the `load` step can be used to invoke those functions like methods.
-Again the [Docker demo](demo/README.md) shows this technique in practice:
+An older version of the [Docker demo](demo/README.md) showed this technique in practice:
 
 ```groovy
 def flow
@@ -608,7 +617,7 @@ node('slave') {
 flow.production()
 ```
 
-where [flow.groovy](https://github.com/jenkinsci/workflow-plugin-pipeline-demo/blob/master/flow.groovy) defines `devQAStaging` and `production` functions (among others) before ending with
+where [flow.groovy](https://github.com/jenkinsci/workflow-plugin-pipeline-demo/blob/641a3491d49570f4f8b9e3e583eb71bad1aa493f/flow.groovy) defines `devQAStaging` and `production` functions (among others) before ending with
 
 ```groovy
 return this;
@@ -617,6 +626,8 @@ return this;
 The subtle part here is that we actually have to do a bit of work with the `node` and `git` steps just to check out a workspace so that we can `load` something.
 In this case `devQAStaging` runs on the same node as the main source code checkout, while `production` runs outside of that block (and in fact allocates a different node).
 
+## Global libraries
+
 Injection of function and class names into a flow before it runs is handled by plugins, and one is bundled with workflow that allows you to get rid of the above boilerplate and keep the whole script (except one “bootstrap” line) in a Git server hosted by Jenkins.
 A [separate document](cps-global-lib/README.md) has details on this system.
 

diff --git a/aggregator/src/test/java/org/jenkinsci/plugins/workflow/CpsScmFlowDefinitionTest.java b/aggregator/src/test/java/org/jenkinsci/plugins/workflow/CpsScmFlowDefinitionTest.java
@@ -0,0 +1,148 @@
+/*
+ * The MIT License
+ *
+ * Copyright 2015 Jesse Glick.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+package org.jenkinsci.plugins.workflow;
+
+import hudson.FilePath;
+import hudson.Launcher;
+import hudson.model.Run;
+import hudson.model.TaskListener;
+import hudson.scm.ChangeLogSet;
+import hudson.scm.NullSCM;
+import hudson.scm.SCMRevisionState;
+import hudson.triggers.SCMTrigger;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.UnsupportedEncodingException;
+import java.net.URLEncoder;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.jenkinsci.plugins.workflow.actions.WorkspaceAction;
+import org.jenkinsci.plugins.workflow.cps.CpsScmFlowDefinition;
+import org.jenkinsci.plugins.workflow.graph.FlowGraphWalker;
+import org.jenkinsci.plugins.workflow.graph.FlowNode;
+import org.jenkinsci.plugins.workflow.job.WorkflowJob;
+import org.jenkinsci.plugins.workflow.job.WorkflowRun;
+import org.jenkinsci.plugins.workflow.steps.scm.GitStep;
+import org.jenkinsci.plugins.workflow.steps.scm.SubversionStepTest;
+import org.junit.Test;
+import static org.junit.Assert.*;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+import org.jvnet.hudson.test.JenkinsRule;
+
+public class CpsScmFlowDefinitionTest {
+
+    @Rule public JenkinsRule r = new JenkinsRule();
+    @Rule public TemporaryFolder tmp = new TemporaryFolder();
+
+    @Test public void basics() throws Exception {
+        WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
+        p.setDefinition(new CpsScmFlowDefinition(new SingleFileSCM("flow.groovy", "echo 'hello from SCM'"), "flow.groovy"));
+        WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0));
+        // TODO currently the log text is in Run.log, but not on FlowStartNode/LogAction, so not visible from Running Steps etc.
+        r.assertLogContains("hello from SCM", b);
+        r.assertLogContains("Staging flow.groovy", b);
+        FlowGraphWalker w = new FlowGraphWalker(b.getExecution());
+        int workspaces = 0;
+        FlowNode n;
+        while ((n = w.next()) != null) {
+            if (n.getAction(WorkspaceAction.class) != null) {
+                workspaces++;
+            }
+        }
+        assertEquals(1, workspaces);
+    }
+
+    private static void git(File repo, String... cmds) throws Exception {
+        List<String> args = new ArrayList<String>();
+        args.add("git");
+        args.addAll(Arrays.asList(cmds));
+        SubversionStepTest.run(repo, args.toArray(new String[args.size()]));
+    }
+
+    /** Otherwise {@link JenkinsRule#waitUntilNoActivity()} is ineffective when we have just pinged a commit notification endpoint. */
+    @Before public void synchronousPolling() {
+        r.jenkins.getDescriptorByType(SCMTrigger.DescriptorImpl.class).synchronousPolling = true;
+    }
+
+    @Test public void changelogAndPolling() throws Exception {
+        File sampleRepo = tmp.newFolder();
+        git(sampleRepo, "init");
+        FileUtils.write(new File(sampleRepo, "flow.groovy"), "echo 'version one'");
+        git(sampleRepo, "add", "flow.groovy");
+        git(sampleRepo, "commit", "--message=init");
+        WorkflowJob p = r.jenkins.createProject(WorkflowJob.class, "p");
+        p.addTrigger(new SCMTrigger("")); // no schedule, use notifyCommit only
+        p.setDefinition(new CpsScmFlowDefinition(new GitStep(sampleRepo.getAbsolutePath()).createSCM(), "flow.groovy"));
+        WorkflowRun b = r.assertBuildStatusSuccess(p.scheduleBuild2(0));
+        r.assertLogContains("Cloning the remote Git repository", b);
+        r.assertLogContains("version one", b);
+        FileUtils.write(new File(sampleRepo, "flow.groovy"), "echo 'version two'");
+        git(sampleRepo, "add", "flow.groovy");
+        git(sampleRepo, "commit", "--message=next");
+        System.out.println(r.createWebClient().goTo("git/notifyCommit?url=" + URLEncoder.encode(sampleRepo.getAbsolutePath(), "UTF-8"), "text/plain").getWebResponse().getContentAsString());
+        r.waitUntilNoActivity();
+        b = p.getLastBuild();
+        assertEquals(2, b.number);
+        r.assertLogContains("Fetching changes from the remote Git repository", b);
+        r.assertLogContains("version two", b);
+        List<ChangeLogSet<? extends ChangeLogSet.Entry>> changeSets = b.getChangeSets();
+        assertEquals(1, changeSets.size());
+        ChangeLogSet<? extends ChangeLogSet.Entry> changeSet = changeSets.get(0);
+        assertEquals(b, changeSet.getRun());
+        assertEquals("git", changeSet.getKind());
+        Iterator<? extends ChangeLogSet.Entry> iterator = changeSet.iterator();
+        assertTrue(iterator.hasNext());
+        ChangeLogSet.Entry entry = iterator.next();
+        assertEquals("[flow.groovy]", entry.getAffectedPaths().toString());
+        assertFalse(iterator.hasNext());
+    }
+
+    // TODO 1.599+ use standard version
+    private static class SingleFileSCM extends NullSCM {
+        private final String path;
+        private final byte[] contents;
+        SingleFileSCM(String path, String contents) throws UnsupportedEncodingException {
+            this.path = path;
+            this.contents = contents.getBytes("UTF-8");
+        }
+        @Override public void checkout(Run<?, ?> build, Launcher launcher, FilePath workspace, TaskListener listener, File changelogFile, SCMRevisionState baseline) throws IOException, InterruptedException {
+            listener.getLogger().println("Staging " + path);
+            OutputStream os = workspace.child(path).write();
+            IOUtils.write(contents, os);
+            os.close();
+        }
+        private Object writeReplace() {
+            return new Object();
+        }
+    }
+
+}
diff --git a/aggregator/src/test/java/org/jenkinsci/plugins/workflow/steps/scm/SubversionStepTest.java b/aggregator/src/test/java/org/jenkinsci/plugins/workflow/steps/scm/SubversionStepTest.java
@@ -57,7 +57,7 @@ public class SubversionStepTest {
     @Rule public JenkinsRule r = new JenkinsRule();
     @Rule public TemporaryFolder tmp = new TemporaryFolder();
 
-    static void run(File cwd, String... cmds) throws Exception {
+    public static void run(File cwd, String... cmds) throws Exception {
         ProcessBuilder pb = new ProcessBuilder(cmds);
         try {
             ProcessBuilder.class.getMethod("inheritIO").invoke(pb);

diff --git a/api/src/main/java/org/jenkinsci/plugins/workflow/flow/FlowDefinition.java b/api/src/main/java/org/jenkinsci/plugins/workflow/flow/FlowDefinition.java
@@ -25,10 +25,15 @@
 package org.jenkinsci.plugins.workflow.flow;
 
 import hudson.ExtensionPoint;
+import hudson.Util;
 import hudson.model.AbstractDescribableImpl;
 import hudson.model.Action;
+import hudson.model.TaskListener;
+import hudson.util.LogTaskListener;
 import java.io.IOException;
 import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * Actual executable script.
@@ -43,7 +48,30 @@ public abstract class FlowDefinition extends AbstractDescribableImpl<FlowDefinit
      * @param actions
      *      Additional parameters to how
      */
-    public abstract FlowExecution create(FlowExecutionOwner handle, List<? extends Action> actions) throws IOException;
+    public /*abstract*/ FlowExecution create(FlowExecutionOwner handle, TaskListener listener, List<? extends Action> actions) throws Exception {
+        if (Util.isOverridden(FlowDefinition.class, getClass(), "create", FlowExecutionOwner.class, List.class)) {
+            return create(handle, actions);
+        } else {
+            throw new NoSuchMethodError();
+        }
+    }
+
+    @Deprecated
+    public FlowExecution create(FlowExecutionOwner handle, List<? extends Action> actions) throws IOException {
+        if (Util.isOverridden(FlowDefinition.class, getClass(), "create", FlowExecutionOwner.class, TaskListener.class, List.class)) {
+            try {
+                return create(handle, new LogTaskListener(Logger.getLogger(FlowDefinition.class.getName()), Level.INFO), actions);
+            } catch (IOException x) {
+                throw x;
+            } catch (RuntimeException x) {
+                throw x;
+            } catch (Exception x) {
+                throw new IOException(x);
+            }
+        } else {
+            throw new NoSuchMethodError();
+        }
+    }
 
     @Override public FlowDefinitionDescriptor getDescriptor() {
         return (FlowDefinitionDescriptor) super.getDescriptor();

diff --git a/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/PushdStep.java b/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/PushdStep.java
@@ -41,12 +41,7 @@ public class PushdStep extends AbstractStepImpl {
     private final String path;
 
     @DataBoundConstructor public PushdStep(String path) {
-        this.path = RelativePathValidator.validate(path);
-    }
-
-    private Object readResolve() {
-        RelativePathValidator.validate(path);
-        return this;
+        this.path = path;
     }
 
     public String getPath() {

diff --git a/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/ReadFileStep.java b/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/ReadFileStep.java
@@ -39,12 +39,12 @@ public final class ReadFileStep extends AbstractStepImpl {
     private String encoding;
 
     @DataBoundConstructor public ReadFileStep(String file) {
-        this.file = RelativePathValidator.validate(file);
-    }
-
-    private Object readResolve() {
-        RelativePathValidator.validate(file);
-        return this;
+        // Normally pointless to verify that this is a relative path, since shell steps can anyway read and write files anywhere on the slave.
+        // Could be necessary in case a plugin installs a {@link LauncherDecorator} which keeps commands inside some kind of jail.
+        // In that case we would need some API to determine that such a jail is in effect and this validation must be enforced.
+        // But just checking the path is anyway not sufficient (due to crafted symlinks); would need to check the final resulting path.
+        // Same for WriteFileStep, PushdStep.
+        this.file = file;
     }
 
     public String getFile() {

diff --git a/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/RelativePathValidator.java b/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/RelativePathValidator.java
diff --git a/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/WriteFileStep.java b/basic-steps/src/main/java/org/jenkinsci/plugins/workflow/steps/WriteFileStep.java
@@ -38,15 +38,10 @@ public final class WriteFileStep extends AbstractStepImpl {
     private String encoding;
 
     @DataBoundConstructor public WriteFileStep(String file, String text) {
-        this.file = RelativePathValidator.validate(file);
+        this.file = file;
         this.text = text;
     }
 
-    private Object readResolve() {
-        RelativePathValidator.validate(file);
-        return this;
-    }
-
     public String getFile() {
         return file;
     }