Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
[JENKINS-51347] Test Status "notrun" being reported as "failed" test
Fix XSL to not include in failure count also skipped tests.
  • Loading branch information
nfalco79 committed May 15, 2018
1 parent 97b19ae commit dd79be0
Show file tree
Hide file tree
Showing 8 changed files with 212 additions and 34 deletions.
Expand Up @@ -32,7 +32,7 @@
/**
* @author Gregory Boissinot
*/
public class CTestInputMetric extends InputMetricXSL {
public class CTest extends InputMetricXSL {

@Override
public InputType getToolType() {
Expand All @@ -41,7 +41,7 @@ public InputType getToolType() {

@Override
public String getToolVersion() {
return "Version N/A";
return "Version 3.x";
}

@Override
Expand Down
Expand Up @@ -52,7 +52,7 @@ public CTestType(String pattern, boolean skipNoTestFiles, boolean failIfNotNew,
public static class CTestTypeDescriptor extends TestTypeDescriptor<CTestType> {

public CTestTypeDescriptor() {
super(CTestType.class, CTestInputMetric.class);
super(CTestType.class, CTest.class);
}

}
Expand Down
Expand Up @@ -22,26 +22,52 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-->
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
<xsl:output method="xml" indent="yes"/>
<xsl:stylesheet version="2.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xunit="http://www.xunit.org">
<xsl:output method="xml" indent="yes" encoding="UTF-8"/>
<xsl:decimal-format decimal-separator="." grouping-separator=","/>

<xsl:function name="xunit:junit-time" as="xs:string">
<xsl:param name="value" as="xs:double?" />

<xsl:variable name="time" as="xs:double">
<xsl:value-of select="translate(string($value),',','.')" />
</xsl:variable>
<xsl:value-of select="format-number($time, '0.000')" />
</xsl:function>

<xsl:function name="xunit:if-empty" as="xs:string">
<xsl:param name="value" as="xs:string?" />
<xsl:param name="default" as="xs:anyAtomicType" />
<xsl:value-of select="if (string($value) != '') then string($value) else $default" />
</xsl:function>

<xsl:function name="xunit:is-empty" as="xs:boolean">
<xsl:param name="value" as="xs:string?" />
<xsl:value-of select="string($value) != ''" />
</xsl:function>

<xsl:template match="/">
<testsuites>
<xsl:variable name="buildName" select="//Site/@BuildName"/>
<xsl:variable name="buildTime" select="(//Site/Testing/EndTestTime - //Site/Testing/StartTestTime)"/>
<xsl:variable name="numberOfTests" select="count(//Site/Testing/Test)"/>
<xsl:variable name="numberOfFailures" select="count(//Site/Testing/Test[@Status!='passed'])"/>
<xsl:variable name="numberOfFailures" select="count(//Site/Testing/Test[@Status='failed'])"/>
<xsl:variable name="numberOfSkipped" select="count(//Site/Testing/Test[@Status='notrun'])"/>
<testsuite name="CTest"
tests="{$numberOfTests}" time="0"
failures="{$numberOfFailures}" errors="0"
skipped="0">
tests="{$numberOfTests}"
time="{xunit:junit-time($buildTime)}"
failures="{$numberOfFailures}"
errors="0"
skipped="{$numberOfSkipped}">
<xsl:for-each select="//Site/Testing/Test">
<xsl:variable name="testName" select="translate(Name, '-', '_')"/>
<xsl:variable name="duration" select="Results/NamedMeasurement[@name='Execution Time']/Value"/>
<xsl:variable name="duration" select="number(xunit:if-empty(Results/NamedMeasurement[@name='Execution Time']/Value, 0))"/>
<xsl:variable name="status" select="@Status"/>
<xsl:variable name="output" select="Results/Measurement/Value"/>
<xsl:variable name="className" select="translate(Path, '/.', '.')"/>
<testcase classname="projectroot{$className}"
name="{$testName}"
time="{$duration}">
time="{xunit:junit-time($duration)}">
<xsl:choose>
<xsl:when test="@Status='passed'"/>
<xsl:when test="@Status='notrun'">
Expand Down
53 changes: 46 additions & 7 deletions src/test/java/org/jenkinsci/plugins/xunit/types/CTestTest.java
@@ -1,16 +1,55 @@
/*
* The MIT License (MIT)
*
* Copyright (c) 2014, Gregory Boissinot, Falco Nikolas
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/
package org.jenkinsci.plugins.xunit.types;

import java.util.Arrays;
import java.util.Collection;

import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;

/**
* @author Gregory Boissinot
*/
@RunWith(Parameterized.class)
public class CTestTest extends AbstractTest {

@Parameters(name = "testcase{1}: {0}")
public static Collection<Object[]> data() {
return Arrays.asList(new Object[][] { { "testcase1", 1 }, //
{ "testcase2", 2 }, //
{ "skipped test are counted as failures", 3 } //
});
}

public CTestTest(String testName, int testNumber) {
super(CTest.class, resolveInput("ctest", testNumber), resolveOutput("ctest", testNumber));
}

@Override
@Test
public void testTestCase1() throws Exception {
convertAndValidate(CTestInputMetric.class, "ctest/testcase1/input.xml", "ctest/testcase1/result.xml");
convertAndValidate(CTestInputMetric.class, "ctest/testcase2/input.xml", "ctest/testcase2/result.xml");
public void verifyXSLT() throws Exception {
super.verifyXSLT();
}

}
}
@@ -1,7 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite name="CTest" tests="11" time="0" failures="4" errors="0" skipped="0">
<testcase classname="projectroot" name="cppcheck" time="1.65417">
<testsuite name="CTest" tests="11" time="3.000" failures="4" errors="0" skipped="0">
<testcase classname="projectroot" name="cppcheck" time="1.654">
<system-out>
Microsoft (R) Visual C++ Express Edition Version 9.0.30729.1.
Copyright (C) Microsoft Corp 2007. All rights reserved.
Expand All @@ -18,18 +18,17 @@
========== Build: 1 succeeded, 0 failed, 1 up-to-date, 0 skipped ==========
</system-out>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="FileSystem_testU" time="0.0970097">
<testcase classname="projectrootE_.foo.sources" name="FileSystem_testU" time="0.097">
<system-out/>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="Logging_testU" time="0.0710068">
<testcase classname="projectrootE_.foo.sources" name="Logging_testU" time="0.071">
<system-out>-- TODO: implement a test if it possible
</system-out>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="MD5_testU" time="0.0730076">
<testcase classname="projectrootE_.foo.sources" name="MD5_testU" time="0.073">
<system-out/>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="ScriptWriter_testU"
time="0.0920086">
<testcase classname="projectrootE_.foo.sources" name="ScriptWriter_testU" time="0.092">
<failure>CMake Error at modules/testU/ScriptWriter-testU.cmake:28 (BAR_script_write_add_to_path):
Unknown CMake command "BAR_script_write_add_to_path".

Expand All @@ -41,19 +40,18 @@

</system-out>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="String_testU" time="0.0980091">
<testcase classname="projectrootE_.foo.sources" name="String_testU" time="0.098">
<system-out/>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="WGET_testU_MD5_fail"
time="0.0760078">
<testcase classname="projectrootE_.foo.sources" name="WGET_testU_MD5_fail" time="0.076">
<failure>-- Download of file://\\wsrvmov003\Softs\packages/BuildSystemTests/WGET-test-wrong-md5.txt failed
with message: [37]"couldn't read a file:// file"
</failure>
<system-out>-- Download of file://\\wsrvmov003\Softs\packages/BuildSystemTests/WGET-test-wrong-md5.txt
failed with message: [37]"couldn't read a file:// file"
</system-out>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="WGET_testU_noMD5" time="0.0820084">
<testcase classname="projectrootE_.foo.sources" name="WGET_testU_noMD5" time="0.082">
<failure>-- Download of file://\\wsrvmov003\Softs\packages/BuildSystemTests/WGET-test-wrong-md5.txt failed
with message: [37]"couldn't read a file:// file"
CMake Error at modules/Logging.cmake:121 (message):
Expand Down Expand Up @@ -81,7 +79,7 @@

</system-out>
</testcase>
<testcase classname="projectrootE_.foo.sources" name="WGET_testU" time="0.115011">
<testcase classname="projectrootE_.foo.sources" name="WGET_testU" time="0.115">
<failure>-- Download of file://\\wsrvmov003\Softs\packages/BuildSystemTests/WGET-test-md5.txt failed with
message: [37]"couldn't read a file:// file"
CMake Error at modules/Logging.cmake:121 (message):
Expand Down Expand Up @@ -109,14 +107,14 @@

</system-out>
</testcase>
<testcase classname="projectroot.tests" name="helloexe_shared" time="0.168016">
<testcase classname="projectroot.tests" name="helloexe_shared" time="0.168">
<system-out>Hello BAR
Hello lib shared
BAR version from hellolib : 2.4.2
BAR version from helloexe : 2.4.2
</system-out>
</testcase>
<testcase classname="projectroot.tests" name="helloexe_static" time="0.124012">
<testcase classname="projectroot.tests" name="helloexe_static" time="0.124">
<system-out>Hello BAR
Hello lib static
BAR version from hellolib : 2.4.2
Expand Down
@@ -1,7 +1,7 @@
<?xml version="1.0"?>
<testsuites>
<testsuite name="CTest" tests="1" time="0" failures="1" errors="0" skipped="0">
<testcase classname="projectroot" name="app_gl_default_sb" time="">
<testsuite name="CTest" tests="1" time="1.000" failures="0" errors="0" skipped="1">
<testcase classname="projectroot" name="app_gl_default_sb" time="0.000">
<skipped/>
<system-out>Run application...
DRY_RUN=yes /var/lib/hudson/workspace/apitrace-mac-test/apps/gl/tri -sb
Expand Down
@@ -0,0 +1,91 @@
<?xml version="1.0" encoding="UTF-8"?>
<Site BuildName="(empty)"
BuildStamp="20180515-1731-Experimental"
Name="(empty)"
Generator="ctest-3.11.0"
CompilerName=""
CompilerVersion=""
OSName="Linux"
Hostname="3tnavBuild"
OSRelease="4.4.0-116-generic"
OSVersion="#140-Ubuntu SMP Mon Feb 12 21:23:04 UTC 2018"
OSPlatform="x86_64"
Is64Bits="1"
VendorString="GenuineIntel"
VendorID="Intel Corporation"
FamilyID="6"
ModelID="44"
ProcessorCacheSize="12288"
NumberOfLogicalCPU="8"
NumberOfPhysicalCPU="8"
TotalVirtualMemory="979"
TotalPhysicalMemory="7982"
LogicalProcessorsPerPhysical="1"
ProcessorClockFrequency="3458"
>
<Testing>
<StartDateTime>May 15 10:31 PDT</StartDateTime>
<StartTestTime>1526405497</StartTestTime>
<TestList>
<Test>./libs/QVtkVisualization/tests/LoggingSinkTests.loggingSinkTest_CallLoggingCallback</Test>
<Test>./tools/simulator/test/simulator.SimulatorTest.readEventFile_mediaDetectedEvent_oneSignalEmitted</Test>
</TestList>
<Test Status="passed">
<Name>LoggingSinkTests.loggingSinkTest_CallLoggingCallback</Name>
<Path>./libs/QVtkVisualization/tests</Path>
<FullName>./libs/QVtkVisualization/tests/LoggingSinkTests.loggingSinkTest_CallLoggingCallback</FullName>
<FullCommandLine>/home/ctc/jenkins/workspace/build_TNAV-dev_Pull-Request/build/libs/QVtkVisualization/tests/QVtkVisualizationTestPublicAPI "--gtest_filter=LoggingSinkTests.loggingSinkTest_CallLoggingCallback"</FullCommandLine>
<Results>
<NamedMeasurement type="numeric/double" name="Execution Time">
<Value>0.0074303</Value>
</NamedMeasurement>
<NamedMeasurement type="numeric/double" name="Processors">
<Value>1</Value>
</NamedMeasurement>
<NamedMeasurement type="text/string" name="Completion Status">
<Value>Completed</Value>
</NamedMeasurement>
<NamedMeasurement type="text/string" name="Command Line">
<Value>/home/ctc/jenkins/workspace/build_TNAV-dev_Pull-Request/build/libs/QVtkVisualization/tests/QVtkVisualizationTestPublicAPI "--gtest_filter=LoggingSinkTests.loggingSinkTest_CallLoggingCallback"</Value>
</NamedMeasurement>
<Measurement>
<Value>Note: Google Test filter = LoggingSinkTests.loggingSinkTest_CallLoggingCallback
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
[----------] 1 test from LoggingSinkTests
[ RUN ] LoggingSinkTests.loggingSinkTest_CallLoggingCallback
[ OK ] LoggingSinkTests.loggingSinkTest_CallLoggingCallback (0 ms)
[----------] 1 test from LoggingSinkTests (0 ms total)

[----------] Global test environment tear-down
[==========] 1 test from 1 test case ran. (0 ms total)
[ PASSED ] 1 test.
</Value>
</Measurement>
</Results>
</Test>
<Test Status="notrun">
<Name>simulator.SimulatorTest.readEventFile_mediaDetectedEvent_oneSignalEmitted</Name>
<Path>./tools/simulator/test</Path>
<FullName>./tools/simulator/test/simulator.SimulatorTest.readEventFile_mediaDetectedEvent_oneSignalEmitted</FullName>
<FullCommandLine></FullCommandLine>
<Results>
<NamedMeasurement type="numeric/double" name="Processors">
<Value>1</Value>
</NamedMeasurement>
<NamedMeasurement type="text/string" name="Completion Status">
<Value>Disabled</Value>
</NamedMeasurement>
<NamedMeasurement type="text/string" name="Command Line">
<Value></Value>
</NamedMeasurement>
<Measurement>
<Value>Disabled</Value>
</Measurement>
</Results>
</Test>
<EndDateTime>May 15 10:37 PDT</EndDateTime>
<EndTestTime>1526405879</EndTestTime>
<ElapsedMinutes>6</ElapsedMinutes>
</Testing>
</Site>
@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<testsuites xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xunit="http://www.xunit.org">
<testsuite name="CTest" tests="2" time="382.000" failures="0" errors="0" skipped="1">
<testcase classname="projectroot.libs.QVtkVisualization.tests" name="LoggingSinkTests.loggingSinkTest_CallLoggingCallback" time="0.007">
<system-out>Note: Google Test filter = LoggingSinkTests.loggingSinkTest_CallLoggingCallback
[==========] Running 1 test from 1 test case.
[----------] Global test environment set-up.
[----------] 1 test from LoggingSinkTests
[ RUN ] LoggingSinkTests.loggingSinkTest_CallLoggingCallback
[ OK ] LoggingSinkTests.loggingSinkTest_CallLoggingCallback (0 ms)
[----------] 1 test from LoggingSinkTests (0 ms total)

[----------] Global test environment tear-down
[==========] 1 test from 1 test case ran. (0 ms total)
[ PASSED ] 1 test.</system-out>
</testcase>
<testcase classname="projectroot.tools.simulator.test"
name="simulator.SimulatorTest.readEventFile_mediaDetectedEvent_oneSignalEmitted"
time="0.000">
<skipped/>
<system-out>Disabled</system-out>
</testcase>
</testsuite>
</testsuites>

0 comments on commit dd79be0

Please sign in to comment.