[JENKINS-44748] Ensure defunct files are removed from populating files (

#56) * Ensure defunct files are removed from populating files * kotlin support * Files Command, with some kotlin
jenkinsci · Jun 12, 2017 · 2214e2f · 2214e2f
1 parent 5e3008e
commit 2214e2f
Show file tree

Hide file tree

Showing 13 changed files with 179 additions and 68 deletions.
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
   <parent>
     <groupId>org.jenkins-ci.plugins</groupId>
     <artifactId>plugin</artifactId>
-    <version>2.29</version>
+    <version>2.30</version>
     <relativePath />
   </parent>
 
@@ -17,6 +17,8 @@
   <properties>
     <jenkins.version>2.32.2</jenkins.version>
     <java.level>8</java.level>
+    <kotlin.version>1.1.2-2</kotlin.version>
+    <maven.javadoc.skip>true</maven.javadoc.skip> <!-- Need to find solution to javadoc/kotlin doc -->
   </properties>
 
   <developers>
@@ -48,6 +50,11 @@
       <version>2.8.9</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.jetbrains.kotlin</groupId>
+      <artifactId>kotlin-stdlib</artifactId>
+      <version>${kotlin.version}</version>
+    </dependency>
   </dependencies>
 
   <scm>
@@ -75,6 +82,41 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-surefire-plugin</artifactId>
       </plugin>
+      <plugin>
+        <groupId>org.jetbrains.kotlin</groupId>
+        <artifactId>kotlin-maven-plugin</artifactId>
+        <version>${kotlin.version}</version>
+        <executions>
+          <execution>
+            <id>compile</id>
+            <phase>process-sources</phase>
+            <goals>
+              <goal>compile</goal>
+            </goals>
+            <configuration>
+              <sourceDirs>
+                <source>src/main/java</source>
+                <source>src/main/kotlin</source>
+                <source>src/main/resources</source>
+              </sourceDirs>
+            </configuration>
+          </execution>
+          <execution>
+            <id>test-compile</id>
+            <phase>process-test-sources</phase>
+            <goals>
+              <goal>test-compile</goal>
+            </goals>
+            <configuration>
+              <sourceDirs>
+                <source>src/test/java</source>
+                <source>src/test/kotlin</source>
+                <source>src/test/resources</source>
+              </sourceDirs>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

diff --git a/src/main/java/hudson/plugins/accurev/AccurevSCM.java b/src/main/java/hudson/plugins/accurev/AccurevSCM.java
@@ -43,8 +43,7 @@
 import java.util.logging.Logger;
 
 /**
- * @author connollys
- * @since 09-Oct-2007 16:17:34
+ * Accurev SCM plugin for Jenkins
  */
 public class AccurevSCM extends SCM {
 
@@ -325,7 +324,7 @@ public void buildEnvVars(AbstractBuild<?, ?> build, Map<String, String> env) {
     }
 
     // TODO: 2.60+ - add @Override.
-    public void buildEnvironment(Run<?,?> build, Map<String, String> env) {
+    public void buildEnvironment(Run<?, ?> build, Map<String, String> env) {
         AbstractModeDelegate delegate = AccurevMode.findDelegate(this);
         delegate.buildEnvVars(build, env);
     }

diff --git a/src/main/java/hudson/plugins/accurev/ParseChangeLog.java b/src/main/java/hudson/plugins/accurev/ParseChangeLog.java
@@ -160,10 +160,7 @@ private List<AccurevTransaction> parseTransactions(XmlPullParser parser, File ch
                     } else if ("version".equalsIgnoreCase(tagName) && currentTransaction != null) {
                         path = parser.getAttributeValue("", "path");
                         if (path != null) {
-                            path = path.replace("\\", "/");
-                            if (path.startsWith("/./")) {
-                                path = path.substring(3);
-                            }
+                            path = AccurevUtils.cleanAccurevPath(path);
                             // currentTransaction.addAffectedPath(path);
 
                         }

diff --git a/src/main/java/hudson/plugins/accurev/cmd/FilesCmd.java b/src/main/java/hudson/plugins/accurev/cmd/FilesCmd.java
@@ -0,0 +1,50 @@
+package hudson.plugins.accurev.cmd;
+
+import hudson.EnvVars;
+import hudson.FilePath;
+import hudson.Launcher;
+import hudson.model.TaskListener;
+import hudson.plugins.accurev.AccurevElement;
+import hudson.plugins.accurev.AccurevLauncher;
+import hudson.plugins.accurev.AccurevSCM;
+import hudson.plugins.accurev.XmlParserFactory;
+import hudson.plugins.accurev.parsers.xml.ParseFiles;
+import hudson.util.ArgumentListBuilder;
+import org.xmlpull.v1.XmlPullParserFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+public class FilesCmd extends Command {
+    private static final Logger logger = Logger.getLogger(Login.class.getName());
+
+    public static List<AccurevElement> checkFiles(
+        final AccurevSCM scm,
+        final AccurevSCM.AccurevServer server,
+        final EnvVars accurevEnv,
+        final FilePath workspace,
+        final TaskListener listener,
+        final Launcher launcher,
+        FilePath file) throws IOException {
+        final String commandDescription = "files command";
+        final ArgumentListBuilder cmd = new ArgumentListBuilder();
+        cmd.add("files");
+        addServer(cmd, server);
+        cmd.add("-fx");
+        cmd.add("-s", scm.getStream());
+        cmd.add("-l", file.getRemote());
+
+        // returns username
+        final ArrayList<AccurevElement> list = new ArrayList<>(1);
+        XmlPullParserFactory parser = XmlParserFactory.getFactory();
+        if (parser == null) throw new IOException("No XML Parser");
+        final Boolean filesFound = AccurevLauncher.runCommand(commandDescription, scm.getAccurevTool(), launcher, cmd, scm.getOptionalLock(),
+            accurevEnv, workspace, listener, logger, parser, new ParseFiles(), list);
+        if (filesFound == null) {
+            throw new IOException("FilesCmd command failed.");
+        }
+        return list;
+    }
+}
diff --git a/src/main/java/hudson/plugins/accurev/cmd/GetAccuRevVersion.java b/src/main/java/hudson/plugins/accurev/cmd/GetAccuRevVersion.java
@@ -2,7 +2,6 @@
 
 import org.apache.log4j.Logger;
 
-import javax.annotation.CheckForNull;
 import java.io.BufferedReader;
 import java.io.IOException;
 import java.io.InputStreamReader;

diff --git a/src/main/java/hudson/plugins/accurev/cmd/PopulateCmd.java b/src/main/java/hudson/plugins/accurev/cmd/PopulateCmd.java
@@ -51,7 +51,7 @@ public boolean populate(AccurevSCM scm, Launcher launcher, TaskListener listener
                             String fromMessage,
                             FilePath workspace,
                             EnvVars accurevEnv,
-                            String files) throws IOException {
+                            FilePath files) throws IOException {
         listener.getLogger().println("Populating " + fromMessage + "...");
         final ArgumentListBuilder cmd = new ArgumentListBuilder();
         cmd.add("pop");
@@ -65,10 +65,10 @@ public boolean populate(AccurevSCM scm, Launcher launcher, TaskListener listener
         cmd.add("-L");
         cmd.add(workspace.getRemote());
 
-        // Add the list files to be populated
+        // Add the file containing list to be populated
         if (files != null) {
             cmd.add("-l");
-            cmd.add(files);
+            cmd.add(files.getRemote());
         }
 
         if (overwrite) cmd.add("-O");

diff --git a/src/main/java/hudson/plugins/accurev/delegates/AbstractModeDelegate.java b/src/main/java/hudson/plugins/accurev/delegates/AbstractModeDelegate.java
@@ -15,15 +15,13 @@
 import org.apache.commons.lang.math.NumberUtils;
 
 import java.io.BufferedReader;
-import java.io.BufferedWriter;
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.text.SimpleDateFormat;
 import java.util.*;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 
 import static java.nio.charset.StandardCharsets.UTF_8;
 
@@ -289,15 +287,15 @@ protected boolean populate(Run<?, ?> build, boolean populateRequired) throws IOE
                 else
                     return false;
             } else {
-                String filePath = getFileRevisionsTobePopulated(build, lastTransaction, getChangeLogStream());
-                logger.info("populate file path " + filePath);
-                if (filePath != null) {
+                FilePath populateFile = getFileRevisionsTobePopulated(lastTransaction, getChangeLogStream());
+                if (populateFile != null) {
+                    logger.info("populate file path " + populateFile.getRemote());
                     if (pop.populate(scm, launcher, listener, server, stream, true, getPopulateFromMessage(), accurevWorkingSpace,
-                        accurevEnv, filePath)) {
+                        accurevEnv, populateFile)) {
                         startDateOfPopulate = pop.get_startDateOfPopulate();
-                        deletePopulateFile(filePath);
+                        deleteTempFile(populateFile);
                     } else {
-                        deletePopulateFile(filePath);
+                        deleteTempFile(populateFile);
                         return false;
                     }
                 }
@@ -413,20 +411,20 @@ private String getLastBuildTransaction(Run<?, ?> build) throws IOException {
      * @throws IOException failed to open file
      */
 
-    private String getFileRevisionsTobePopulated(Run<?, ?> build, int lastTransaction, String stream) throws IOException {
+    private FilePath getFileRevisionsTobePopulated(int lastTransaction, String stream) throws IOException, InterruptedException {
         List<AccurevTransaction> transactions = History.getTransactionsAfterLastTransaction(scm, server, accurevEnv,
             accurevWorkingSpace, listener, launcher, stream, lastTransaction);
+        if (transactions.isEmpty()) {
+            return null;
+        }
         // collect all the files from the list of transactions and remove duplicates from the list of files.
-        List<String> fileRevisions = transactions
-            .stream()
-            .filter(t -> t != null && !(t.getAction().equals("defunct")))
+        Set<String> fileRevisions = new HashSet<>();
+        transactions.stream()
+            .filter(t -> t != null && !t.getAction().equals("defunct") && !t.getAffectedPaths().isEmpty())
             .map(AccurevTransaction::getAffectedPaths)
-            .flatMap(Collection::stream)
-            .collect(Collectors.toList())
-            .parallelStream()
-            .distinct()
-            .collect(Collectors.toList());
-        return (!fileRevisions.isEmpty()) ? getPopulateFilePath(build, fileRevisions) : null;
+            .forEach(fileRevisions::addAll);
+        if (fileRevisions.isEmpty()) return null;
+        return checkFilesOnStream(getPopulateFilePath(fileRevisions));
     }
 
     /**
@@ -435,35 +433,35 @@ private String getFileRevisionsTobePopulated(Run<?, ?> build, int lastTransactio
      * @param fileRevisions current file revisions
      * @return String
      */
-    private String getPopulateFilePath(Run<?, ?> build, List<String> fileRevisions) {
-        BufferedWriter bw = null;
-        File populateFile;
-        String filepath = null;
-        try {
-            populateFile = new File(build.getParent().getRootDir(), POPULATE_FILES);
-            filepath = populateFile.getAbsolutePath();
-            logger.info("populate file path is " + populateFile.getAbsolutePath());
-            bw = Files.newBufferedWriter(populateFile.toPath(), UTF_8);
-            for (String filePath : fileRevisions) {
-                bw.write(filePath);
-                bw.newLine();
-            }
-        } catch (IOException exe) {
-            logger.info("Exception happend to write in a file." + exe);
-        } finally {
+    private FilePath getPopulateFilePath(Set<String> fileRevisions) throws IOException, InterruptedException {
+        FilePath populateFiles = accurevWorkingSpace.createTextTempFile("PopulateFiles", ".txt", String.join("\n", fileRevisions));
+        populateFiles.chmod(0600);
+        return populateFiles;
+    }
+
+    private FilePath checkFilesOnStream(FilePath filePath) throws IOException, InterruptedException {
+        Set<String> fileRevisions = new HashSet<>();
+        List<AccurevElement> accurevElements = FilesCmd.checkFiles(scm, server, accurevEnv, accurevWorkingSpace, listener, launcher, filePath);
+        accurevElements.stream()
+            .filter(e -> e != null && !e.getStatus().matches(".*(defunct|no such elem).*"))
+            .map(AccurevElement::getLocation)
+            .forEach(fileRevisions::add);
+        for (String str : fileRevisions) {
+            listener.getLogger().print(str);
+        }
+        filePath.write(String.join("\n", fileRevisions), "UTF-8");
+        return filePath;
+    }
+
+    private void deleteTempFile(FilePath tempFile) throws InterruptedException, IOException {
+        if (tempFile != null) {
             try {
-                if (bw != null)
-                    bw.close();
+                tempFile.delete();
             } catch (IOException ex) {
-                logger.info("Exception happend to close the buffered writer." + ex);
+                if (tempFile.exists()) {
+                    listener.getLogger().println("[WARNING] temp file " + tempFile + " not deleted");
+                }
             }
         }
-        return filepath;
-    }
-
-    private void deletePopulateFile(String filePath) {
-        File populateFile = new File(filePath);
-        boolean deleted = populateFile.delete();
-        logger.info("temporary file deleted " + deleted);
     }
 }
diff --git a/src/main/java/hudson/plugins/accurev/parsers/xml/ParseHistory.java b/src/main/java/hudson/plugins/accurev/parsers/xml/ParseHistory.java
@@ -3,6 +3,7 @@
 import hudson.plugins.accurev.AccurevLauncher.ICmdOutputXmlParser;
 import hudson.plugins.accurev.AccurevLauncher.UnhandledAccurevCommandOutput;
 import hudson.plugins.accurev.AccurevTransaction;
+import hudson.plugins.accurev.AccurevUtils;
 import hudson.plugins.accurev.ParseChangeLog;
 import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;
@@ -31,10 +32,7 @@ public Boolean parse(XmlPullParser parser, List<AccurevTransaction> context) thr
                     // parse path & convert it to standard format
                     String path = parser.getAttributeValue("", "path");
                     if (path != null) {
-                        path = path.replace("\\", "/");
-                        if (path.startsWith("/./")) {
-                            path = path.substring(3);
-                        }
+                        path = AccurevUtils.cleanAccurevPath(path);
                     }
                     resultTransaction.addAffectedPath(path);
                 }

diff --git a/src/main/java/hudson/plugins/accurev/parsers/xml/ParseUpdate.java b/src/main/java/hudson/plugins/accurev/parsers/xml/ParseUpdate.java
@@ -1,6 +1,7 @@
 package hudson.plugins.accurev.parsers.xml;
 
 import hudson.plugins.accurev.AccurevLauncher;
+import hudson.plugins.accurev.AccurevUtils;
 import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;
 
@@ -16,11 +17,7 @@ public Boolean parse(XmlPullParser parser, List<String> context) throws AccurevL
             if (parser.getEventType() == XmlPullParser.START_TAG && "element".equalsIgnoreCase(parser.getName())) {
                 String path = parser.getAttributeValue("", "location");
                 if (path != null) {
-                    path = path.replace("\\", "/");
-                    if (path.startsWith("/./")) {
-                        path = path.substring(3);
-                    }
-                    context.add(path);
+                    context.add(AccurevUtils.cleanAccurevPath(path));
                 }
             }
         }

diff --git a/src/main/kotlin/hudson/plugins/accurev/AccurevElement.kt b/src/main/kotlin/hudson/plugins/accurev/AccurevElement.kt
@@ -0,0 +1,3 @@
+package hudson.plugins.accurev
+
+data class AccurevElement(val location: String, val status: String)
diff --git a/src/main/kotlin/hudson/plugins/accurev/AccurevUtils.kt b/src/main/kotlin/hudson/plugins/accurev/AccurevUtils.kt
@@ -0,0 +1,7 @@
+package hudson.plugins.accurev
+
+open class AccurevUtils {
+    companion object {
+        @JvmStatic fun cleanAccurevPath(str: String) = str.replace("\\", "/").removePrefix("/./")
+    }
+}
diff --git a/src/main/kotlin/hudson/plugins/accurev/parsers/xml/ParseFiles.kt b/src/main/kotlin/hudson/plugins/accurev/parsers/xml/ParseFiles.kt
@@ -0,0 +1,21 @@
+package hudson.plugins.accurev.parsers.xml
+
+import hudson.plugins.accurev.AccurevElement
+import hudson.plugins.accurev.AccurevLauncher
+import hudson.plugins.accurev.AccurevUtils.Companion.cleanAccurevPath
+import org.xmlpull.v1.XmlPullParser
+
+class ParseFiles : AccurevLauncher.ICmdOutputXmlParser<Boolean, ArrayList<AccurevElement>> {
+    override fun parse(parser: XmlPullParser?, context: ArrayList<AccurevElement>?): Boolean {
+        while (parser?.next() != XmlPullParser.END_DOCUMENT) {
+            if (parser?.eventType == XmlPullParser.START_TAG) {
+                if ("element".equals(parser.name, ignoreCase = true)) {
+                    val location = cleanAccurevPath(parser.getAttributeValue("", "location"))
+                    val status = parser.getAttributeValue("", "status")
+                    context?.add(AccurevElement(location, status))
+                }
+            }
+        }
+        return context != null
+    }
+}
diff --git a/src/main/webapp/help/project/stream.html b/src/main/webapp/help/project/stream.html
@@ -1,5 +1,5 @@
 <div>
-    <p>Enter the <b>Stream</b> name / <b>Workspace</b> name by typing.</p>
-    <p>Input field offers auto complete suggestion</p>
-    <p><b>Note:</b> Build will be generated from <b>Stream</b> or <b>Workspace</b>, as per the user's selection.</p>
+  <p>Enter the <b>Stream</b> name / <b>Workspace</b> name by typing.</p>
+  <p>Input field offers auto complete suggestion</p>
+  <p><b>Note:</b> Build will be generated from <b>Stream</b> or <b>Workspace</b>, as per the user's selection.</p>
 </div>