Skip to content

Commit

Permalink
[JENKINS-17960] Indicate if tests don't go to plan
Browse files Browse the repository at this point in the history
  • Loading branch information
@kinow
kinow committed Mar 5, 2014
1 parent c6f236c commit 9e60ea5
Show file tree
Hide file tree
Showing 6 changed files with 104 additions and 135 deletions.
102 changes: 42 additions & 60 deletions src/main/java/org/tap4j/plugin/TapParser.java
View file
Expand Up @@ -51,56 +51,18 @@ public class TapParser {
/** Prints the logs to the web server's console / log files */
private static final Logger log = Logger.getLogger(TapParser.class
.getName());
private Boolean outputTapToConsole;
private Boolean enableSubtests;
private Boolean todoIsFailure;
private final Boolean outputTapToConsole;
private final Boolean enableSubtests;
private final Boolean todoIsFailure;

/** Build's logger to print logs as part of build's console output */
private PrintStream logger;
private boolean parserErrors;
private final PrintStream logger;
private final Boolean includeCommentDiagnostics;
private final Boolean validateNumberOfTests;
private final Boolean planRequired;

private boolean hasFailedTests;
private boolean includeCommentDiagnostics;
private boolean validateNumberOfTests;
private boolean planRequired;

/**
* @deprecated
*/
@Deprecated
public TapParser(Boolean outputTapToConsole, Boolean enableSubtests, Boolean todoIsFailure, PrintStream logger) {
this.outputTapToConsole = outputTapToConsole;
this.enableSubtests = enableSubtests;
this.todoIsFailure = todoIsFailure;
this.logger = logger;
this.parserErrors = false;
this.hasFailedTests = false;
}

/**
* @deprecated
*/
@Deprecated
public TapParser(Boolean outputTapToConsole, Boolean enableSubtests, Boolean todoIsFailure, Boolean includeCommentDiagnostics, PrintStream logger) {
this.outputTapToConsole = outputTapToConsole;
this.enableSubtests = enableSubtests;
this.todoIsFailure = todoIsFailure;
this.logger = logger;
this.parserErrors = false;
this.includeCommentDiagnostics = includeCommentDiagnostics;
}

/**
* @deprecated
*/
public TapParser(Boolean outputTapToConsole, Boolean enableSubtests, Boolean todoIsFailure, Boolean includeCommentDiagnostics, Boolean validateNumberOfTests, PrintStream logger) {
this.outputTapToConsole = outputTapToConsole;
this.enableSubtests = enableSubtests;
this.todoIsFailure = todoIsFailure;
this.logger = logger;
this.parserErrors = false;
this.includeCommentDiagnostics = includeCommentDiagnostics;
this.validateNumberOfTests = validateNumberOfTests;
}
private boolean parserErrors;

public TapParser(Boolean outputTapToConsole, Boolean enableSubtests, Boolean todoIsFailure,
Boolean includeCommentDiagnostics, Boolean validateNumberOfTests, Boolean planRequired,
Expand All @@ -115,11 +77,39 @@ public TapParser(Boolean outputTapToConsole, Boolean enableSubtests, Boolean tod
this.planRequired = planRequired;
}

public boolean hasParserErrors() {
public Boolean hasParserErrors() {
return this.parserErrors;
}

public Boolean getOutputTapToConsole() {
return outputTapToConsole;
}

public Boolean getTodoIsFailure() {
return todoIsFailure;
}

public boolean getParserErrors() {
return parserErrors;
}

public Boolean getIncludeCommentDiagnostics() {
return includeCommentDiagnostics;
}

public boolean hasFailedTests() {
public Boolean getValidateNumberOfTests() {
return validateNumberOfTests;
}

public Boolean getPlanRequired() {
return planRequired;
}

public Boolean getEnableSubtests() {
return enableSubtests;
}

public boolean hasFailedTests() {
return this.hasFailedTests;
}

Expand All @@ -133,25 +123,17 @@ public TapResult parse(FilePath[] results, AbstractBuild<?, ?> build) {
for (FilePath path : results) {
File tapFile = new File(path.getRemote());
if (!tapFile.isFile()) {
log("'" + tapFile.getAbsolutePath()
+ "' points to an invalid test report");
log("'" + tapFile.getAbsolutePath() + "' points to an invalid test report");
continue; // move to next file
} else {
log("Processing '" + tapFile.getAbsolutePath() + "'");
}
try {
log("Parsing TAP test result [" + tapFile + "].");

final Tap13Parser parser;
parser = new Tap13Parser("UTF-8", enableSubtests, planRequired);
final Tap13Parser parser = new Tap13Parser("UTF-8", enableSubtests, planRequired);
final TestSet testSet = parser.parseFile(tapFile);

if (this.validateNumberOfTests) {
if (testSet.getPlan().getLastTestNumber() != testSet.getNumberOfTestResults()) {
throw new ParserException("Number of tests results didn't go to plan");
}
}

if (testSet.containsNotOk() || testSet.containsBailOut()) {
this.hasFailedTests = Boolean.TRUE;
}
Expand All @@ -176,7 +158,7 @@ public TapResult parse(FilePath[] results, AbstractBuild<?, ?> build) {
}
}
//final TapResult testResult = new TapResult(UUID.randomUUID().toString(), build, testSets);
final TapResult testResult = new TapResult("TAP Test Results", build, testSets, this.todoIsFailure, this.includeCommentDiagnostics);
final TapResult testResult = new TapResult("TAP Test Results", build, testSets, this.todoIsFailure, this.includeCommentDiagnostics, this.validateNumberOfTests);
return testResult;
}

Expand Down
59 changes: 40 additions & 19 deletions src/main/java/org/tap4j/plugin/TapPublisher.java
View file
Expand Up @@ -49,8 +49,12 @@

import org.apache.commons.lang.BooleanUtils;
import org.kohsuke.stapler.DataBoundConstructor;
import org.tap4j.model.Plan;
import org.tap4j.model.TestResult;
import org.tap4j.model.TestSet;
import org.tap4j.plugin.model.TestSetMap;
import org.tap4j.plugin.util.Constants;
import org.tap4j.util.StatusValues;

/**
* Publishes TAP results in Jenkins builds.
Expand All @@ -71,23 +75,6 @@ public class TapPublisher extends Recorder implements MatrixAggregatable {
private final Boolean validateNumberOfTests;
private final Boolean planRequired;

/**
* Kept for backward compatibility. To be removed in next major release.
* @deprecated
*/
public TapPublisher(String testResults,
Boolean failIfNoResults,
Boolean failedTestsMarkBuildAsFailure,
Boolean outputTapToConsole,
Boolean enableSubtests,
Boolean discardOldReports,
Boolean todoIsFailure,
Boolean includeCommentDiagnostics,
Boolean validateNumberOfTests) {
this(testResults, failIfNoResults, failedTestsMarkBuildAsFailure, outputTapToConsole, enableSubtests,
discardOldReports, todoIsFailure, includeCommentDiagnostics, validateNumberOfTests, Boolean.TRUE);
}

@DataBoundConstructor
public TapPublisher(String testResults,
Boolean failIfNoResults,
Expand Down Expand Up @@ -273,12 +260,21 @@ public boolean perform(AbstractBuild<?, ?> build, Launcher launcher,
TapBuildAction action = new TapBuildAction(build, testResult);
build.getActions().add(action);
if (testResult.hasParseErrors()) {
listener.getLogger().println("TAP parse errors found in the build. Marking build as UNSTABLE");
build.setResult(Result.UNSTABLE);
}
if (this.getValidateNumberOfTests()) {
if (!this.validateNumberOfTests(testResult.getTestSets())) {
listener.getLogger().println("Not all test cases were executed according to the test set plan. Marking build as UNSTABLE");
build.setResult(Result.UNSTABLE);
}
}
if (testResult.getFailed() > 0) {
if(this.getFailedTestsMarkBuildAsFailure()) {
listener.getLogger().println("There are failed test cases and the job is configured to mark the build as failure. Marking build as FAILURE");
build.setResult(Result.FAILURE);
} else {
listener.getLogger().println("There are failed test cases. Marking build as UNSTABLE");
build.setResult(Result.UNSTABLE);
}
}
Expand All @@ -291,6 +287,31 @@ public boolean perform(AbstractBuild<?, ?> build, Launcher launcher,
}

/**
* Iterates through the list of test sets and validates its plans and
* test results.
*
* @param testSets
* @return <true> if there are any test case that doesn't follow the plan
*/
private boolean validateNumberOfTests(List<TestSetMap> testSets) {
for (TestSetMap testSetMap : testSets) {
TestSet testSet = testSetMap.getTestSet();
Plan plan = testSet.getPlan();
if (plan != null) {
int planned = plan.getLastTestNumber();
int totalWithSuccess = 0;
for (TestResult tr : testSet.getTestResults()) {
if (tr.getStatus() == StatusValues.OK)
totalWithSuccess++;
}
if (planned != totalWithSuccess)
return false;
}
}
return true;
}

/**
* @param build
* @param logger
* @return
Expand All @@ -309,7 +330,7 @@ private TapResult loadResults(AbstractBuild<?, ?> owner, PrintStream logger) {
} catch (Exception e) {
e.printStackTrace(logger);

tr = new TapResult("", owner, Collections.<TestSetMap>emptyList(), getTodoIsFailure(), getIncludeCommentDiagnostics());
tr = new TapResult("", owner, Collections.<TestSetMap>emptyList(), getTodoIsFailure(), getIncludeCommentDiagnostics(), getValidateNumberOfTests());
tr.setOwner(owner);
return tr;
}
Expand Down Expand Up @@ -449,7 +470,7 @@ public String getDisplayName() {
* @see hudson.tasks.BuildStepDescriptor#isApplicable(java.lang.Class)
*/
@Override
public boolean isApplicable(Class<? extends AbstractProject> jobType) {
public boolean isApplicable(@SuppressWarnings("rawtypes") Class<? extends AbstractProject> jobType) {
return Boolean.TRUE;
}

Expand Down
38 changes: 8 additions & 30 deletions src/main/java/org/tap4j/plugin/TapResult.java
View file
Expand Up @@ -80,44 +80,18 @@ public class TapResult implements ModelObject, Serializable {
private String name;
private Boolean todoIsFailure;
private Boolean includeCommentDiagnostics;
private Boolean validateNumberOfTests;

/**
* @deprecated since JENKINS-15401
* @param name
* @param owner
* @param testSets
*/
@Deprecated
public TapResult(String name, AbstractBuild<?, ?> owner,
List<TestSetMap> testSets) {
this.name = name;
this.build = owner;
this.testSets = this.filterTestSet(testSets);
this.parseErrorTestSets = this.filterParseErrorTestSets(testSets);
this.todoIsFailure = true;
}

/**
* @deprecated
*/
@Deprecated
public TapResult(String name, AbstractBuild<?, ?> owner,
List<TestSetMap> testSets, Boolean todoIsFailure) {
this.name = name;
this.build = owner;
this.testSets = this.filterTestSet(testSets);
this.parseErrorTestSets = this.filterParseErrorTestSets(testSets);
this.todoIsFailure = todoIsFailure;
}

public TapResult(String name, AbstractBuild<?, ?> owner,
List<TestSetMap> testSets, Boolean todoIsFailure, Boolean includeCommentDiagnostics) {
List<TestSetMap> testSets, Boolean todoIsFailure, Boolean includeCommentDiagnostics ,
Boolean validateNumberOfTests) {
this.name = name;
this.build = owner;
this.testSets = this.filterTestSet(testSets);
this.parseErrorTestSets = this.filterParseErrorTestSets(testSets);
this.todoIsFailure = todoIsFailure;
this.includeCommentDiagnostics= includeCommentDiagnostics;
this.validateNumberOfTests = validateNumberOfTests;
}

/**
Expand All @@ -134,6 +108,10 @@ public Boolean getIncludeCommentDiagnostics() {
return (includeCommentDiagnostics == null) ? true : includeCommentDiagnostics;
}

public Boolean getValidateNumberOfTests() {
return (validateNumberOfTests == null) ? false : validateNumberOfTests;
}

/**
* @param testSets
* Untiltered test sets
Expand Down
9 changes: 4 additions & 5 deletions src/main/java/org/tap4j/plugin/model/TapStreamResult.java
View file
Expand Up @@ -34,7 +34,6 @@
import java.util.Collections;
import java.util.List;

import org.apache.commons.lang.StringUtils;
import org.kohsuke.stapler.StaplerRequest;
import org.kohsuke.stapler.StaplerResponse;
import org.kohsuke.stapler.export.Exported;
Expand All @@ -56,13 +55,13 @@ public class TapStreamResult extends TabulatedResult {

public TapStreamResult(AbstractBuild<?, ?> owner, TapResult tapResult) {
this.owner = owner;
this.tapResult = tapResult;
for(TestSetMap tsm : tapResult.getTestSets()) {
TestSet ts = tsm.getTestSet();
for(org.tap4j.model.TestResult tr : ts.getTestResults()) {
this.children.add(new TapTestResultResult(owner, tsm, tr, this.tapResult.getTodoIsFailure()));
this.children.add(new TapTestResultResult(owner, tsm, tr, tapResult.getTodoIsFailure(), tapResult.getIncludeCommentDiagnostics(), tapResult.getValidateNumberOfTests()));
}
}
this.tapResult = tapResult;
}

/* (non-Javadoc)
Expand Down Expand Up @@ -155,7 +154,7 @@ public List<TestResult> getFailedTests2() {
TestSet ts = tsm.getTestSet();
for(org.tap4j.model.TestResult tr : ts.getTestResults()) {
if(tr.getStatus() == StatusValues.NOT_OK) {
failedTests.add(new TapTestResultResult(owner, tsm, tr, this.tapResult.getTodoIsFailure()));
failedTests.add(new TapTestResultResult(owner, tsm, tr, this.tapResult.getTodoIsFailure(), tapResult.getIncludeCommentDiagnostics(), tapResult.getValidateNumberOfTests()));
}
}
}
Expand Down Expand Up @@ -201,7 +200,7 @@ private TapTestResultResult getTapTestResultResult(String name) {
if(tsm.getFileName().equals(fileName)) {
TestSet ts = tsm.getTestSet();
org.tap4j.model.TestResult desired = ts.getTestResult(Integer.parseInt(testNumber));
return new TapTestResultResult(owner, tsm, desired, this.tapResult.getTodoIsFailure());
return new TapTestResultResult(owner, tsm, desired, this.tapResult.getTodoIsFailure(), tapResult.getIncludeCommentDiagnostics(), tapResult.getValidateNumberOfTests());
}
}

Expand Down

0 comments on commit 9e60ea5

Please sign in to comment.