Skip to content

Commit

Permalink
Merge pull request #14 from Vlatombe/JENKINS-29894
Browse files Browse the repository at this point in the history
[JENKINS-29894] Allow to specify includesPatternFile
  • Loading branch information
Vlatombe committed Aug 19, 2015
2 parents f11a8e4 + fce5c45 commit 7b5683d
Show file tree
Hide file tree
Showing 23 changed files with 587 additions and 27 deletions.
7 changes: 4 additions & 3 deletions demo/JENKINS_HOME/jobs/flow/config.xml
Expand Up @@ -9,15 +9,16 @@
git url: 'https://github.com/jenkinsci/parallel-test-executor-plugin-sample'
archive 'pom.xml, src/'
}
def splits = splitTests([$class: 'CountDrivenParallelism', size: 5])
def splits = splitTests(parallelism: [$class: 'CountDrivenParallelism', size: 5], generateInclusions: true)
def branches = [:]
for (int i = 0; i < splits.size(); i++) {
def exclusions = splits.get(i);
def split = splits.get(i);
branches["split${i}"] = {
node('standard') {
sh 'rm -rf *'
unarchive mapping: ['pom.xml' : '.', 'src/' : '.']
writeFile file: 'exclusions.txt', text: exclusions.join("\n")
writeFile file: (split.includes ? 'inclusions.txt' : 'exclusions.txt'), text: split.list.join("\n")
writeFile file: (split.includes ? 'exclusions.txt' : 'inclusions.txt'), text: ''
sh 'mvn -B clean test -Dmaven.test.failure.ignore'
step([$class: 'JUnitResultArchiver', testResults: 'target/surefire-reports/*.xml'])
}
Expand Down
1 change: 1 addition & 0 deletions demo/JENKINS_HOME/jobs/main/config.xml
Expand Up @@ -17,6 +17,7 @@
<size>5</size>
</parallelism>
<testJob>sub</testJob>
<includesPatternFile>inclusions.txt</includesPatternFile>
<patternFile>exclusions.txt</patternFile>
<testReportFiles>**/target/surefire-reports/</testReportFiles>
<doNotArchiveTestResults>false</doNotArchiveTestResults>
Expand Down
11 changes: 11 additions & 0 deletions pom.xml
Expand Up @@ -27,11 +27,22 @@
<artifactId>junit</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>1.9.5</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
<artifactId>workflow-step-api</artifactId>
<version>${workflow.version}</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>script-security</artifactId>
<version>1.13</version>
<optional>true</optional>
</dependency>
<dependency> <!-- For ease of demonstrating parallel-test-executor-plugin-sample -->
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>git</artifactId>
Expand Down
@@ -0,0 +1,38 @@
package org.jenkinsci.plugins.parallel_test_executor;

import org.jenkinsci.plugins.scriptsecurity.sandbox.whitelists.Whitelisted;

import java.io.Serializable;
import java.util.Collections;
import java.util.List;

/**
* A list of file name patterns to include or exclude
*/
public class InclusionExclusionPattern implements Serializable {
@Whitelisted
public boolean isIncludes() {
return includes;
}

@Whitelisted
public List<String> getList() {
return Collections.unmodifiableList(list);
}

private final boolean includes;
private final List<String> list;

public InclusionExclusionPattern(List<String> list, boolean includes) {
this.list = list;
this.includes = includes;
}

@Override
public String toString() {
return "InclusionExclusionPattern{" +
"includes=" + includes +
", list=" + list +
'}';
}
}
@@ -0,0 +1,96 @@
package org.jenkinsci.plugins.parallel_test_executor;

import com.google.common.collect.Lists;
import hudson.Extension;
import hudson.FilePath;
import hudson.model.AbstractBuild;
import hudson.model.Action;
import hudson.model.FileParameterValue;
import hudson.model.ParametersAction;
import hudson.model.TaskListener;
import hudson.plugins.parameterizedtrigger.AbstractBuildParameterFactory;
import hudson.plugins.parameterizedtrigger.AbstractBuildParameterFactoryDescriptor;
import hudson.plugins.parameterizedtrigger.AbstractBuildParameters;
import hudson.plugins.parameterizedtrigger.FileBuildParameterFactory;
import org.kohsuke.stapler.DataBoundConstructor;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.util.List;
import java.util.logging.Logger;

/**
* Essentially a copy-paste of {@link hudson.plugins.parameterizedtrigger.BinaryFileParameterFactory} that takes a
* list of mappings "name -> filePattern" to generate parameters.
*/
public class MultipleBinaryFileParameterFactory extends AbstractBuildParameterFactory {
public static class ParameterBinding {
public ParameterBinding(String parameterName, String filePattern) {
this.parameterName = parameterName;
this.filePattern = filePattern;
}
public final String parameterName;
public final String filePattern;
}

private final List<ParameterBinding> parametersList;
private final FileBuildParameterFactory.NoFilesFoundEnum noFilesFoundAction;

@DataBoundConstructor
public MultipleBinaryFileParameterFactory(List<ParameterBinding> parametersList, FileBuildParameterFactory.NoFilesFoundEnum noFilesFoundAction) {
this.parametersList = parametersList;
this.noFilesFoundAction = noFilesFoundAction;
}

public MultipleBinaryFileParameterFactory(List<ParameterBinding> parametersList) {
this(parametersList, FileBuildParameterFactory.NoFilesFoundEnum.SKIP);
}

public FileBuildParameterFactory.NoFilesFoundEnum getNoFilesFoundAction() {
return noFilesFoundAction;
}

@Override
public List<AbstractBuildParameters> getParameters(AbstractBuild<?, ?> build, TaskListener listener) throws IOException, InterruptedException, AbstractBuildParameters.DontTriggerException {
List<AbstractBuildParameters> result = Lists.newArrayList();
int totalFiles = 0;
for (final ParameterBinding parameterBinding : parametersList) {
// save them into the master because FileParameterValue might need files after the slave workspace have disappeared/reused
FilePath target = new FilePath(build.getRootDir()).child("parameter-files");
int k = build.getWorkspace().copyRecursiveTo(parameterBinding.filePattern, target);
totalFiles += k;
if (k > 0) {
for (final FilePath f : target.list(parameterBinding.filePattern)) {
LOGGER.fine("Triggering build with " + f.getName());

result.add(new AbstractBuildParameters() {
@Override
public Action getAction(AbstractBuild<?, ?> build, TaskListener listener) throws IOException, InterruptedException, DontTriggerException {
assert f.getChannel() == null; // we copied files locally. This file must be local to the master
FileParameterValue fv = new FileParameterValue(parameterBinding.parameterName, new File(f.getRemote()), f.getName());
return new ParametersAction(fv);
}
});
}
}
}
if (totalFiles ==0) {
noFilesFoundAction.failCheck(listener);
}

return result;
}


@Extension
public static class DescriptorImpl extends AbstractBuildParameterFactoryDescriptor {
@Override
public String getDisplayName() {
return "Multiple Binary Files (not meant to be used)";
}
}

private static final Logger LOGGER = Logger.getLogger(MultipleBinaryFileParameterFactory.class.getName());
}
Expand Up @@ -4,6 +4,7 @@
import hudson.Extension;
import hudson.FilePath;
import hudson.Launcher;
import hudson.Util;
import hudson.model.*;
import hudson.plugins.parameterizedtrigger.*;
import hudson.tasks.BuildStepDescriptor;
Expand All @@ -15,9 +16,9 @@
import hudson.tasks.test.TestResult;
import org.kohsuke.stapler.AncestorInPath;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.DataBoundSetter;
import org.kohsuke.stapler.QueryParameter;

import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
Expand All @@ -26,6 +27,8 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.io.Charsets;

import javax.annotation.CheckForNull;

/**
* @author Kohsuke Kawaguchi
*/
Expand All @@ -36,6 +39,7 @@ public class ParallelTestExecutor extends Builder {

private String testJob;
private String patternFile;
private String includesPatternFile;
private String testReportFiles;
private boolean doNotArchiveTestResults = false;
private List<AbstractBuildParameters> parameters;
Expand All @@ -62,6 +66,16 @@ public String getPatternFile() {
return patternFile;
}

@CheckForNull
public String getIncludesPatternFile() {
return includesPatternFile;
}

@DataBoundSetter
public void setIncludesPatternFile(String includesPatternFile) {
this.includesPatternFile = Util.fixEmpty(includesPatternFile);
}

public String getTestReportFiles() {
return testReportFiles;
}
Expand Down Expand Up @@ -101,13 +115,14 @@ public int compareTo(Knapsack that) {
public boolean perform(AbstractBuild<?, ?> build, Launcher launcher, BuildListener listener) throws InterruptedException, IOException {
FilePath dir = build.getWorkspace().child("test-splits");
dir.deleteRecursive();
List<List<String>> splits = findTestSplits(parallelism, build, listener);
List<InclusionExclusionPattern> splits = findTestSplits(parallelism, build, listener, includesPatternFile != null);
for (int i = 0; i < splits.size(); i++) {
OutputStream os = dir.child("split." + i + ".txt").write();
InclusionExclusionPattern pattern = splits.get(i);
OutputStream os = dir.child("split." + i + "." + (pattern.isIncludes() ? "include" : "exclude") + ".txt").write();
try {
PrintWriter pw = new PrintWriter(new OutputStreamWriter(os, Charsets.UTF_8));
for (String exclusion : splits.get(i)) {
pw.println(exclusion);
for (String filePattern : pattern.getList()) {
pw.println(filePattern);
}
pw.close();
} finally {
Expand All @@ -124,11 +139,11 @@ public boolean perform(AbstractBuild<?, ?> build, Launcher launcher, BuildListen
return true;
}

static List<List<String>> findTestSplits(Parallelism parallelism, Run<?,?> build, TaskListener listener) {
static List<InclusionExclusionPattern> findTestSplits(Parallelism parallelism, Run<?,?> build, TaskListener listener, boolean generateInclusions) {
TestResult tr = findPreviousTestResult(build, listener);
if (tr == null) {
listener.getLogger().println("No record available, so executing everything in one place");
return Collections.singletonList(Collections.<String>emptyList());
return Collections.singletonList(new InclusionExclusionPattern(Collections.<String>emptyList(), false));
} else {

Map<String/*fully qualified class name*/, TestClass> data = new HashMap<String, TestClass>();
Expand Down Expand Up @@ -173,15 +188,17 @@ static List<List<String>> findTestSplits(Parallelism parallelism, Run<?,?> build
listener.getLogger().printf("%d test classes (%dms) divided into %d sets. Min=%dms, Average=%dms, Max=%dms, stddev=%dms\n",
data.size(), total, n, min, average, max, stddev);

List<List<String>> r = new ArrayList<List<String>>();
List<InclusionExclusionPattern> r = new ArrayList<InclusionExclusionPattern>();
for (int i = 0; i < n; i++) {
Knapsack k = knapsacks.get(i);
List<String> exclusions = new ArrayList<String>();
r.add(exclusions);
boolean shouldIncludeElements = generateInclusions && i != 0;
List<String> elements = new ArrayList<String>();
r.add(new InclusionExclusionPattern(elements, shouldIncludeElements));
for (TestClass d : sorted) {
if (d.knapsack == k) continue;
exclusions.add(d.getSourceFileName(".java"));
exclusions.add(d.getSourceFileName(".class"));
if (shouldIncludeElements == (d.knapsack == k)) {
elements.add(d.getSourceFileName(".java"));
elements.add(d.getSourceFileName(".class"));
}
}
}
return r;
Expand Down Expand Up @@ -218,11 +235,16 @@ public Action getAction(AbstractBuild<?, ?> build, TaskListener listener) throws
}

// actual logic of child process triggering is left up to the parameterized build
List<MultipleBinaryFileParameterFactory.ParameterBinding> parameterBindings = new ArrayList<MultipleBinaryFileParameterFactory.ParameterBinding>();
parameterBindings.add(new MultipleBinaryFileParameterFactory.ParameterBinding(getPatternFile(), "test-splits/split.*.exclude.txt"));
if (includesPatternFile != null) {
parameterBindings.add(new MultipleBinaryFileParameterFactory.ParameterBinding(getIncludesPatternFile(), "test-splits/split.*.include.txt"));
}
MultipleBinaryFileParameterFactory factory = new MultipleBinaryFileParameterFactory(parameterBindings);
BlockableBuildTriggerConfig config = new BlockableBuildTriggerConfig(
testJob,
blocking,
Collections.<AbstractBuildParameterFactory>singletonList(
new BinaryFileParameterFactory(getPatternFile(), "test-splits/split.*.txt")),
Collections.<AbstractBuildParameterFactory>singletonList(factory),
parameterList
);

Expand Down
Expand Up @@ -4,12 +4,18 @@
import hudson.Extension;
import hudson.model.Run;
import hudson.model.TaskListener;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;

import org.jenkinsci.plugins.workflow.steps.AbstractStepDescriptorImpl;
import org.jenkinsci.plugins.workflow.steps.AbstractStepImpl;
import org.jenkinsci.plugins.workflow.steps.AbstractSynchronousStepExecution;
import org.jenkinsci.plugins.workflow.steps.Step;
import org.jenkinsci.plugins.workflow.steps.StepContextParameter;
import org.kohsuke.stapler.DataBoundConstructor;
import org.kohsuke.stapler.DataBoundSetter;

/**
* Allows the splitting logic to be accessed from a workflow.
Expand All @@ -18,6 +24,8 @@ public final class SplitStep extends AbstractStepImpl {

private final Parallelism parallelism;

private boolean generateInclusions;

@DataBoundConstructor public SplitStep(Parallelism parallelism) {
this.parallelism = parallelism;
}
Expand All @@ -26,6 +34,13 @@ public Parallelism getParallelism() {
return parallelism;
}

public boolean isGenerateInclusions() { return generateInclusions; }

@DataBoundSetter
public void setGenerateInclusions(boolean generateInclusions) {
this.generateInclusions = generateInclusions;
}

@Extension public static final class DescriptorImpl extends AbstractStepDescriptorImpl {

public DescriptorImpl() {
Expand All @@ -42,14 +57,22 @@ public DescriptorImpl() {

}

public static final class Execution extends AbstractSynchronousStepExecution<List<List<String>>> {
public static final class Execution extends AbstractSynchronousStepExecution<List<?>> {

@Inject private SplitStep step;
@StepContextParameter private Run<?,?> build;
@StepContextParameter private TaskListener listener;

@Override protected List<List<String>> run() throws Exception {
return ParallelTestExecutor.findTestSplits(step.parallelism, build, listener);
@Override protected List<?> run() throws Exception {
if (step.generateInclusions) {
return ParallelTestExecutor.findTestSplits(step.parallelism, build, listener, step.generateInclusions);
} else {
List<List<String>> result = new ArrayList<List<String>>();
for (InclusionExclusionPattern pattern : ParallelTestExecutor.findTestSplits(step.parallelism, build, listener, step.generateInclusions)) {
result.add(pattern.getList());
}
return result;
}
}

}
Expand Down
Expand Up @@ -11,6 +11,9 @@ f.entry(title:"Test job to run", field:"testJob") {
f.entry(title:"Exclusion file name in the test job", field:"patternFile") {
f.textbox()
}
f.entry(title:"Optional inclusion file name in the test job", field:"includesPatternFile") {
f.textbox()
}
f.entry(title:"Degree of parallelism", field:"parallelism") {
f.hetero_radio(field:"parallelism", descriptors:Jenkins.instance.getDescriptorList(Parallelism.class))
}
Expand Down

0 comments on commit 7b5683d

Please sign in to comment.