Navigation Menu

Skip to content

Commit

Permalink
[JENKINS-15863][JENKINS-13936] Write HTML content in file when XML is
Browse files Browse the repository at this point in the history 
parsed.
HTML contents of the result was saved in memory before to be written in
files. To avoid OOM, I write the file when the XML si parsed.
  • Loading branch information
@antoine-aumjaud
antoine-aumjaud committed Aug 28, 2014
1 parent c48dfae commit 7ea548e
Show file tree
Hide file tree
Showing 7 changed files with 84 additions and 117 deletions.
3 changes: 1 addition & 2 deletions src/main/java/hudson/plugins/fitnesse/CompoundFitnesseResults.java
View file
Expand Up @@ -16,7 +16,6 @@ public static FitnesseResults createFor(List<FitnesseResults> resultsList) {
String page = "All Results";
String resultsDate = null;
int right = 0, wrong = 0, ignored = 0, exceptions = 0;
String content = null;

for (FitnesseResults fitnesseResults : resultsList) {
if (resultsDate == null) {
Expand All @@ -28,7 +27,7 @@ public static FitnesseResults createFor(List<FitnesseResults> resultsList) {
exceptions += fitnesseResults.getExceptionCount();
}

Counts counts = new Counts(page, resultsDate, right, wrong, ignored, exceptions, content);
Counts counts = new Counts(page, resultsDate, right, wrong, ignored, exceptions, null);
return new CompoundFitnesseResults(resultsList, counts);
}

Expand Down
65 changes: 1 addition & 64 deletions src/main/java/hudson/plugins/fitnesse/FitnesseResultsRecorder.java
View file
Expand Up @@ -9,17 +9,14 @@
import hudson.model.Result;
import hudson.model.AbstractBuild;
import hudson.model.AbstractProject;
import hudson.plugins.fitnesse.NativePageCounts.Counts;
import hudson.tasks.BuildStep;
import hudson.tasks.BuildStepDescriptor;
import hudson.tasks.BuildStepMonitor;
import hudson.tasks.Publisher;
import hudson.tasks.Recorder;
import hudson.util.FormValidation;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;
Expand All @@ -28,7 +25,6 @@
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;

import javax.servlet.ServletException;
import javax.xml.transform.TransformerException;
Expand Down Expand Up @@ -146,12 +142,8 @@ public FitnesseResults getResults(PrintStream logger, FilePath resultsFile,

logger.println("Parsing results... ");
NativePageCountsParser pageCountsParser = new NativePageCountsParser();
NativePageCounts pageCounts = pageCountsParser
.parse(resultsInputStream);

logger.println("all-content: " + pageCounts.getAllContents().size());
NativePageCounts pageCounts = pageCountsParser.parse(resultsInputStream, logger, rootDir.getAbsolutePath() + "/");
logger.println("resultsFile: " + getFitnessePathToXmlResultsIn());
writeFitnesseResultFiles(logger, pageCounts, rootDir);

logger.println("Got results: " + pageCounts.getSummary());
return new FitnesseResults(pageCounts);
Expand Down Expand Up @@ -211,59 +203,4 @@ public String getDisplayName() {
return "Publish Fitnesse results report";
}
}

/**
* Gets a parsed fitnesse result and writes it to separate file. Putting the
* fitnesse result in a separate file as performance reasons. E.g. for a
* huge Test-Suite the actual fitnesse result can grow up to several MB.
* First implementation of fitnesse plugin has stored the result to the
* build.xml. This was very handy to present the result but slowed down
* jenkins since a request to the fitnesse result leat to putting the entire
* build.xml into the memory. With this function the fitnesse result is only
* load to memory if the user clicks on it.
*
* @param logger
* @param pageCounts
* @param rootDir
*/
private void writeFitnesseResultFiles(PrintStream logger,
NativePageCounts pageCounts, File rootDir) {
String rootDirName = rootDir.getAbsolutePath() + "/";
logger.println("write fitnesse results to: " + rootDirName);
Map<String, String> allContent = pageCounts.getAllContents();
logger.println("allContent:\n" + allContent.keySet());
// iterate over all fitnesse tests in a suite
for (Counts iCount : pageCounts.getAllCounts()) {
String name = iCount.page;
String content = allContent.get(name);
if (null == content) {
logger.println("could not find content for page: " + name);
continue;
}
BufferedWriter out = null;
String fileName = rootDirName + name;
try {
// Create separate file for every test in a suite
FileWriter fstream = new FileWriter(fileName);
out = new BufferedWriter(fstream);
out.write(content);
// Just store the path to the filename.
// Any help welcome how to
// restore the path of the saved file when the user wants to see the details
iCount.contentFile = fileName;
} catch (IOException e) {
logger.println("error while writing to out file" + fileName
+ "\n" + e.toString());
} finally {
if (null != out) {
try {
out.close();
} catch (IOException e) {
logger.println("could not close out stream: " + "\n"
+ fileName + e.toString());
}
}
}
}
}
}
95 changes: 64 additions & 31 deletions src/main/java/hudson/plugins/fitnesse/NativePageCounts.java
View file
@@ -1,5 +1,9 @@
package hudson.plugins.fitnesse;

import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
Expand Down Expand Up @@ -28,7 +32,17 @@ public class NativePageCounts extends DefaultHandler {
SUMMARY, DETAIL });

private Counts summary;
private Map<String, Counts> allCounts = new HashMap<String, Counts>();
private final Map<String, Counts> allCounts = new HashMap<String, Counts>();

private final String rootDirName;
private final PrintStream logger;

public NativePageCounts(PrintStream logger, String rootDirName) {
this.logger = logger;
this.rootDirName = rootDirName;
logger.println("Write fitnesse results to: " + rootDirName);
}

/**
* Stores the actual fitnesse results. We do not want to merge them into
* build.xml since the results may have size of several MB. E.g. in a suite
Expand All @@ -39,34 +53,24 @@ public class NativePageCounts extends DefaultHandler {
* The allContents is read later on. The content is written to separate
* files.
*/
private Map<String, String> allContents = new HashMap<String, String>();

@Override
public void startElement(String uri, String localName, String qName,
Attributes attributes) {
if (COUNTABLE.contains(qName)) {
String page = attributes.getValue(PAGE);
String pseudoPage = attributes.getValue(PSEUDO_PAGE);
String content = attributes.getValue(CONTENT);
String targetPage = page == null || page.equals("") ? pseudoPage : page;

Counts counts = new Counts(
page == null || page.equals("") ? pseudoPage : page,
qName.equals(SUMMARY) ? "" : resultsDateOf(attributes
.getValue(APPROX_RESULT_DATE)),
targetPage,
qName.equals(SUMMARY) ? "" : resultsDateOf(attributes.getValue(APPROX_RESULT_DATE)),
Integer.parseInt(attributes.getValue(RIGHT)),
Integer.parseInt(attributes.getValue(WRONG)),
Integer.parseInt(attributes.getValue(IGNORED)),
Integer.parseInt(attributes.getValue(EXCEPTIONS)), "" // see
// above,
// do
// not
// put
// fitnesse-results
// into
// build.xml
Integer.parseInt(attributes.getValue(EXCEPTIONS)),
writeFitnesseResultFiles(targetPage, attributes.getValue(CONTENT))
);
allContents.put(page, content); // see above, save actual result for
// later usage

if (qName.equals(SUMMARY))
summary = counts;
allCounts.put(counts.page, counts);
Expand All @@ -84,14 +88,6 @@ public int size() {
return allCounts.size();
}

/**
*
* @return the fitnesse results (html-content)
*/
public Map<String, String> getAllContents() {
return allContents;
}

public Counts getSummary() {
if (summary != null && summary.right == 0 && summary.wrong == 0
&& summary.ignored == 0 && summary.exceptions == 0) {
Expand Down Expand Up @@ -138,20 +134,19 @@ static final class Counts {
public final int wrong;
public final int ignored;
public final int exceptions;
public final String content;
public String content; // TODO this useless field !
// stores the file-path where to find the actual fitnesse result (html)
// does anybody has a better idea: how to restore the path when the user clicks on the details link?
public String contentFile;
public final String contentFile;

public Counts(String page, String resultsDate, int right, int wrong,
int ignored, int exceptions, String content) {
public Counts(String page, String resultsDate, int right, int wrong, int ignored,
int exceptions, String contentFile) {
this.page = page;
this.resultsDate = resultsDate;
this.right = right;
this.wrong = wrong;
this.ignored = ignored;
this.exceptions = exceptions;
this.content = content;
this.contentFile = contentFile;
}

public Date resultsDateAsDate() throws ParseException {
Expand All @@ -166,4 +161,42 @@ public String toString() {
}

}

/**
* Gets a parsed fitnesse result and writes it to separate file. Putting the
* fitnesse result in a separate file as performance reasons. E.g. for a huge
* Test-Suite the actual fitnesse result can grow up to several MB. First
* implementation of fitnesse plugin has stored the result to the build.xml.
* This was very handy to present the result but slowed down jenkins since a
* request to the fitnesse result leat to putting the entire build.xml into
* the memory. With this function the fitnesse result is only load to memory
* if the user clicks on it.
*/
private String writeFitnesseResultFiles(String pageName, String htmlContent) {
if (null == htmlContent) {
logger.println("Could not find content for page: " + pageName);
return null;
}
BufferedWriter out = null;
String fileName = rootDirName + pageName + ".htm";
try {
// Create separate file for every test in a suite
FileWriter fstream = new FileWriter(fileName);
out = new BufferedWriter(fstream);
out.write(htmlContent);
// Just store the path to the filename.
return fileName;
} catch (IOException e) {
logger.println("Error while writing to out file" + fileName + "\n" + e.toString());
} finally {
if (null != out) {
try {
out.close();
} catch (IOException e) {
logger.println("Could not close out stream: " + fileName + "\n" + e.toString());
}
}
}
return null;
}
}
6 changes: 4 additions & 2 deletions src/main/java/hudson/plugins/fitnesse/NativePageCountsParser.java
View file
Expand Up @@ -2,6 +2,7 @@

import java.io.IOException;
import java.io.InputStream;
import java.io.PrintStream;

import javax.xml.transform.Result;
import javax.xml.transform.Transformer;
Expand All @@ -11,8 +12,9 @@

public class NativePageCountsParser {

public NativePageCounts parse(InputStream inputStream) throws TransformerException, IOException {
NativePageCounts fitnessePageCounts = new NativePageCounts();
public NativePageCounts parse(InputStream inputStream, PrintStream logger, String rootDirName)
throws TransformerException, IOException {
NativePageCounts fitnessePageCounts = new NativePageCounts(logger, rootDirName);
SAXResult intermediateResult = new SAXResult(fitnessePageCounts);
transformRawResults(inputStream, intermediateResult);
return fitnessePageCounts;
Expand Down
8 changes: 2 additions & 6 deletions src/test/java/hudson/plugins/fitnesse/FitnesseResultsRecorderTest.java
View file
Expand Up @@ -3,9 +3,7 @@
import hudson.FilePath;
import hudson.tasks.test.TestResult;

import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.util.Collection;

import org.junit.Assert;
Expand All @@ -20,10 +18,9 @@ public void getResultsShouldReadFromFilePath() throws Exception {
String resultsFile = "src/test/resources/hudson/plugins/fitnesse/fitnesse-test-results.xml";
FitnesseResultsRecorder recorder = new FitnesseResultsRecorder(
resultsFile);
ByteArrayOutputStream log = new ByteArrayOutputStream();
FilePath resultFile = new FilePath(new File(
System.getProperty("user.dir"))).child(resultsFile);
Assert.assertNotNull(recorder.getResults(new PrintStream(log),
Assert.assertNotNull(recorder.getResults(System.out,
resultFile, new File(resultFile.getParent().getBaseName())));
}

Expand All @@ -33,13 +30,12 @@ public void getPatternResults() throws Exception {
String resultsFile = "src/test/resources/hudson/plugins/fitnesse/fitnesse-*-results.xml";
FitnesseResultsRecorder recorder = new FitnesseResultsRecorder(
resultsFile);
ByteArrayOutputStream log = new ByteArrayOutputStream();
FilePath[] resultFiles = recorder.getResultFiles(new FilePath(new File(
System.getProperty("user.dir"))));
Assert.assertNotNull(resultFiles);
Assert.assertEquals(2, resultFiles.length);

FitnesseResults results = recorder.getResults(new PrintStream(log),
FitnesseResults results = recorder.getResults(System.out,
resultFiles, new File(resultFiles[0].getParent().getParent().getBaseName()));
Assert.assertNotNull(results);
Assert.assertTrue(results.hasChildren());
Expand Down
8 changes: 4 additions & 4 deletions src/test/java/hudson/plugins/fitnesse/NativePageCountsParserTest.java
View file
Expand Up @@ -59,7 +59,7 @@ public void transformRawResultsShouldIgnoreBOM() throws Exception {

@Test
public void parserShouldCollectFinalCounts() throws Exception {
NativePageCounts testResults = fitnesseParser.parse(toInputStream(RESULTS));
NativePageCounts testResults = fitnesseParser.parse(toInputStream(RESULTS), System.out, "./");
Assert.assertEquals(2, testResults.size());
Assert.assertEquals("SuiteBlah", testResults.getSummary().page);
Assert.assertEquals(5, testResults.getSummary().right);
Expand All @@ -70,7 +70,7 @@ public void parserShouldCollectFinalCounts() throws Exception {

@Test
public void parserShouldCollectContents() throws Exception {
NativePageCounts testResults = fitnesseParser.parse(toInputStream(RESULTS));
NativePageCounts testResults = fitnesseParser.parse(toInputStream(RESULTS), System.out, "./");
Assert.assertEquals(2, testResults.size());
Assert.assertEquals("SuiteBlah", testResults.getSummary().page);
Assert.assertEquals(1, testResults.getDetailsContents().size());
Expand All @@ -79,7 +79,7 @@ public void parserShouldCollectContents() throws Exception {
@Test
public void parserShouldCollectAllCountsFromSuiteFile() throws Exception {
InputStream sampleXml = getClass().getResourceAsStream("fitnesse-suite-results.xml");
NativePageCounts testResults = fitnesseParser.parse(sampleXml);
NativePageCounts testResults = fitnesseParser.parse(sampleXml, System.out, "./");
Assert.assertEquals(15, testResults.size());
Assert.assertEquals("SuiteBranchMain", testResults.getSummary().page);
Assert.assertEquals(6, testResults.getSummary().right);
Expand All @@ -91,7 +91,7 @@ public void parserShouldCollectAllCountsFromSuiteFile() throws Exception {
@Test
public void parserShouldCollectAllCountsFromSingleTestFile() throws Exception {
InputStream sampleXml = getClass().getResourceAsStream("fitnesse-test-results.xml");
NativePageCounts testResults = fitnesseParser.parse(sampleXml);
NativePageCounts testResults = fitnesseParser.parse(sampleXml, System.out, "./");
Assert.assertEquals(1, testResults.size());
Assert.assertEquals("TestDecisionTable", testResults.getSummary().page);
Assert.assertEquals(16, testResults.getSummary().right);
Expand Down

0 comments on commit 7ea548e

Please sign in to comment.