[JENKINS-4769] Memory consumption is huge

- New data format introduced, it should be backward compatible. - Only statistics per language are stored in the publisher. - The legacy structures are still stored too but with no data (they are null). This is to be able to load the legacy data that were stored using old version of the plugin. - The report data can loaded from the original files after they are needed. - Trend graph updated to use the new language statistics. - Report summary updated to use the new language statistics. - The plugin now consumes much less memory and is much faster most of the time. - Only statistics are used during results publication at build time, displaying trend graph and displaying report summary. Only small statistics are deserialized. - Complete report is loaded only when report details page is displayed.
jenkinsci · Dec 29, 2013 · 284e6ea · 284e6ea
1 parent 5493d93
commit 284e6ea
Show file tree

Hide file tree

Showing 10 changed files with 455 additions and 55 deletions.
diff --git a/src/main/java/hudson/plugins/sloccount/ReportSummary.java b/src/main/java/hudson/plugins/sloccount/ReportSummary.java
@@ -1,10 +1,12 @@
 package hudson.plugins.sloccount;
 
 import hudson.plugins.sloccount.model.Language;
+import hudson.plugins.sloccount.model.SloccountLanguageStatistics;
 import hudson.plugins.sloccount.model.SloccountReport;
 import hudson.plugins.sloccount.util.StringUtil;
 
 import java.io.Serializable;
+import java.util.List;
 
 /**
  *
@@ -16,36 +18,36 @@ private ReportSummary(){
 
     }
 
-    public static String createReportSummary(SloccountReport report, SloccountReport previous){
+    public static String createReportSummary(List<SloccountLanguageStatistics> current,
+    		List<SloccountLanguageStatistics> previous){
         StringBuilder builder = new StringBuilder();
 
-        if(report != null){
-
+        if(current != null){
             String strLines     = Messages.Sloccount_ReportSummary_Lines();
             String strFiles     = Messages.Sloccount_ReportSummary_Files();
             String strAnd       = Messages.Sloccount_ReportSummary_and();
             String strIn        = Messages.Sloccount_ReportSummary_in();
             String strLanguages = Messages.Sloccount_ReportSummary_Languages();
 
             builder.append("<a href=\"" + SloccountBuildAction.URL_NAME + "\">");
-            builder.append(report.getLineCountString());
+            builder.append(getLineCount(current));
 
             if(previous != null) {
-                printDifference(report.getLineCount(), previous.getLineCount(), builder);
+                printDifference(getLineCount(current), getLineCount(previous), builder);
             }
 
             builder.append(" " + strLines + "</a> " + strIn + " ");
-            builder.append(report.getFileCountString());
+            builder.append(getFileCount(current));
 
             if(previous != null) {
-                printDifference(report.getFileCount(), previous.getFileCount(), builder);
+                printDifference(getFileCount(current), getFileCount(previous), builder);
             }
 
             builder.append(" " + strFiles + " " + strAnd + " ");
-            builder.append(report.getLanguageCountString());
+            builder.append(getLanguageCount(current));
 
             if(previous != null) {
-                printDifference(report.getLanguageCount(), previous.getLanguageCount(), builder);
+                printDifference(getLanguageCount(current), getLanguageCount(previous), builder);
             }
 
             builder.append(" " + strLanguages + ".");
@@ -54,18 +56,19 @@ public static String createReportSummary(SloccountReport report, SloccountReport
         return builder.toString();
     }
 
-    public static String createReportSummaryDetails(SloccountReport report, SloccountReport previous){
+    public static String createReportSummaryDetails(List<SloccountLanguageStatistics> current,
+    		List<SloccountLanguageStatistics> previous){
 
         StringBuilder builder = new StringBuilder();
 
-        if(report != null){
+        if(current != null){
 
-            for(Language language : report.getLanguages()){
+            for(SloccountLanguageStatistics language : current){
 
-                Language previousLanguage = null;
+            	SloccountLanguageStatistics previousLanguage = null;
 
                 if(previous != null) {
-                    previousLanguage = previous.getLanguage(language.getName());
+                    previousLanguage = getLanguage(previous, language.getName());
                 }
 
                 appendLanguageDetails(language, previousLanguage, builder);
@@ -75,7 +78,8 @@ public static String createReportSummaryDetails(SloccountReport report, Sloccoun
         return builder.toString();
     }
 
-    private static void appendLanguageDetails(Language language, Language previous, StringBuilder builder){
+    private static void appendLanguageDetails(SloccountLanguageStatistics current,
+    		SloccountLanguageStatistics previous, StringBuilder builder){
 
         String strLines     = Messages.Sloccount_ReportSummary_Lines();
         String strFiles     = Messages.Sloccount_ReportSummary_Files();
@@ -85,18 +89,18 @@ private static void appendLanguageDetails(Language language, Language previous,
         builder.append("<a href=\"");
         builder.append(SloccountBuildAction.URL_NAME);
         builder.append("/languageResult/");
-        builder.append(language.getName());
+        builder.append(current.getName());
         builder.append("\">");
-        builder.append(language.getName());
+        builder.append(current.getName());
         builder.append("</a> : ");
-        builder.append(language.getLineCountString());
+        builder.append(current.getLineCount());
         if(previous != null){
-            printDifference(language.getLineCount(), previous.getLineCount(), builder);
+            printDifference(current.getLineCount(), previous.getLineCount(), builder);
         }
         builder.append(" " + strLines + " " + strIn + " ");
-        builder.append(language.getFileCountString());
+        builder.append(current.getFileCount());
         if(previous != null){
-            printDifference(language.getFileCount(), previous.getFileCount(), builder);
+            printDifference(current.getFileCount(), previous.getFileCount(), builder);
         }
         builder.append(" " + strFiles + ".</li>");
     }
@@ -121,4 +125,42 @@ else if(difference == 0)
             builder.append(")");
         }
     }
+
+    private static int getLineCount(List<SloccountLanguageStatistics> statistics)
+    {
+        int lineCount = 0;
+
+        for(SloccountLanguageStatistics it : statistics) {
+            lineCount += it.getLineCount();
+        }
+
+        return lineCount;    	
+    }
+
+    private static int getFileCount(List<SloccountLanguageStatistics> statistics)
+    {
+        int fileCount = 0;
+
+        for(SloccountLanguageStatistics it : statistics) {
+            fileCount += it.getFileCount();
+        }
+
+        return fileCount; 	
+    }
+
+    private static int getLanguageCount(List<SloccountLanguageStatistics> statistics)
+    {
+        return statistics.size();
+    }
+
+    private static SloccountLanguageStatistics getLanguage(List<SloccountLanguageStatistics> statistics, String name)
+    {
+    	for(SloccountLanguageStatistics it : statistics) {
+            if(it.getName().equals(name)) {
+            	return it;
+            }
+        }
+
+    	return new SloccountLanguageStatistics(name, 0, 0);
+    }
 }
diff --git a/src/main/java/hudson/plugins/sloccount/SloccountBuildAction.java b/src/main/java/hudson/plugins/sloccount/SloccountBuildAction.java
@@ -2,8 +2,12 @@
 
 import hudson.model.AbstractBuild;
 import hudson.model.Action;
+import hudson.plugins.sloccount.model.SloccountLanguageStatistics;
 import hudson.plugins.sloccount.model.SloccountReport;
+
 import java.io.Serializable;
+import java.util.List;
+
 import org.kohsuke.stapler.StaplerProxy;
 
 /**
@@ -40,7 +44,8 @@ public String getSummary(){
 
         if(this.result != null){
 
-            retVal =  ReportSummary.createReportSummary(this.result.getReport(), this.getPreviousReport());
+            retVal =  ReportSummary.createReportSummary(this.result.getStatistics(),
+            		this.getPreviousStatistics());
         }
 
         return retVal;
@@ -51,7 +56,8 @@ public String getDetails(){
         String retVal = "";
 
         if(this.result != null){
-            retVal =  ReportSummary.createReportSummaryDetails(this.result.getReport(), this.getPreviousReport());
+            retVal =  ReportSummary.createReportSummaryDetails(this.result.getStatistics(),
+            		this.getPreviousStatistics());
         }
 
         return retVal;
@@ -61,12 +67,12 @@ public SloccountResult getResult(){
         return this.result;
     }
 
-    private SloccountReport getPreviousReport(){
+    private List<SloccountLanguageStatistics> getPreviousStatistics(){
         SloccountResult previous = this.getPreviousResult();
         if(previous == null){
             return null;
         }else{
-           return previous.getReport();
+           return previous.getStatistics();
         }
     }
 

diff --git a/src/main/java/hudson/plugins/sloccount/SloccountChartBuilder.java b/src/main/java/hudson/plugins/sloccount/SloccountChartBuilder.java
@@ -1,10 +1,12 @@
 package hudson.plugins.sloccount;
 
 import hudson.plugins.sloccount.model.Language;
+import hudson.plugins.sloccount.model.SloccountLanguageStatistics;
 import hudson.plugins.sloccount.model.SloccountReport;
 import hudson.util.ChartUtil.NumberOnlyBuildLabel;
 import hudson.util.DataSetBuilder;
 import hudson.util.ShiftedCategoryAxis;
+
 import java.awt.Color;
 import java.io.Serializable;
 
@@ -68,10 +70,9 @@ private static CategoryDataset buildDataset(SloccountBuildAction lastAction){
         do{
             SloccountResult result = action.getResult();
             if(result != null){
-                SloccountReport report = result.getReport();
                 NumberOnlyBuildLabel buildLabel = new NumberOnlyBuildLabel(action.getBuild());
 
-                for(Language l : report.getLanguages()){
+                for(SloccountLanguageStatistics l : result.getStatistics()){
                     builder.add(l.getLineCount(), l.getName(), buildLabel);
                 }
             }

diff --git a/src/main/java/hudson/plugins/sloccount/SloccountPublisher.java b/src/main/java/hudson/plugins/sloccount/SloccountPublisher.java
@@ -7,13 +7,19 @@
 import hudson.model.Action;
 import hudson.model.BuildListener;
 import hudson.model.Result;
+import hudson.plugins.sloccount.model.SloccountPublisherReport;
 import hudson.plugins.sloccount.model.SloccountReport;
 import hudson.plugins.sloccount.model.SloccountParser;
+import hudson.remoting.VirtualChannel;
 import hudson.tasks.BuildStepMonitor;
 import hudson.tasks.Recorder;
+
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.io.Serializable;
+import java.util.List;
+import java.io.File;
 
 import org.kohsuke.stapler.DataBoundConstructor;
 
@@ -23,6 +29,9 @@
  */
 public class SloccountPublisher extends Recorder implements Serializable {
 
+    /** Subdirectory of build results directory where source files are stored. */
+    public static final String BUILD_SUBDIR = "sloccount-plugin";
+
     private static final String DEFAULT_PATTERN = "**/sloccount.sc";
     private static final String DEFAULT_ENCODING = "UTF-8";
 
@@ -49,23 +58,19 @@ protected boolean canContinue(final Result result) {
 
     @Override
     public boolean perform(AbstractBuild<?,?> build, Launcher launcher, BuildListener listener){
-
-        SloccountResult result;
-
         FilePath workspace = build.getWorkspace();
         PrintStream logger = listener.getLogger();
         SloccountParser parser = new SloccountParser(this.getRealEncoding(), this.getRealPattern(), logger);
-        SloccountReport report;
+        SloccountPublisherReport report;
 
         try{
             if(this.canContinue(build.getResult())){
                 report = workspace.act(parser);
             }else{
                 // generate an empty report
                 // TODO: Replace this empty report with the last valid one?
-                report = new SloccountReport();
+                report = new SloccountPublisherReport();
             }
-
         }catch(IOException ioe){
             ioe.printStackTrace(logger);
             return false;
@@ -75,15 +80,67 @@ public boolean perform(AbstractBuild<?,?> build, Launcher launcher, BuildListene
             return false;
         }
 
-        result = new SloccountResult(report, build);
+        SloccountResult result = new SloccountResult(report.getStatistics(),
+        		getRealEncoding(), null, build);
 
         SloccountBuildAction buildAction = new SloccountBuildAction(build, result);
 
         build.addAction(buildAction);
-
+
+        try{
+            copyFilesToBuildDirectory(report.getSourceFiles(),
+                    build.getRootDir(), launcher.getChannel());
+        }catch(IOException e){
+            e.printStackTrace(logger);
+            return false;
+        }catch(InterruptedException e){
+            e.printStackTrace(logger);
+            return false;
+        }
+
         return true;
     }
 
+    /**
+     * Copy files to a build results directory. The copy of a file will be
+     * stored in plugin's subdirectory and a hashcode of its absolute path will
+     * be used in its name prefix to distinguish files with the same names from
+     * different directories.
+     * 
+     * @param sourceFiles
+     *            the files to copy
+     * @param rootDir
+     *            the root directory where build results are stored
+     * @param channel
+     *            the communication channel
+     * @throws IOException
+     *             if something fails
+     * @throws InterruptedException
+     *             if something fails
+     */
+    private void copyFilesToBuildDirectory(List<File> sourceFiles,
+            File rootDir, VirtualChannel channel) throws IOException,
+            InterruptedException{
+        File destDir = new File(rootDir, BUILD_SUBDIR);
+
+        if(!destDir.exists() && !destDir.mkdir()){
+            throw new IOException(
+                    "Creating directory for copy of workspace files failed: "
+                            + destDir.getAbsolutePath());
+        }
+
+        for(File sourceFile : sourceFiles){
+            File masterFile = new File(destDir, Integer.toHexString(sourceFile
+                    .hashCode()) + "_" + sourceFile.getName());
+
+            if(!masterFile.exists()){
+                FileOutputStream outputStream = new FileOutputStream(masterFile);
+                new FilePath(channel, sourceFile.getAbsolutePath())
+                        .copyTo(outputStream);
+            }
+        }
+    }
+
     public BuildStepMonitor getRequiredMonitorService() {
         return BuildStepMonitor.BUILD;
     }