Merge branch 'master' into JENKINS-26781 and fixing @SInCE TODO.

jenkinsci · Apr 14, 2015 · b62a07c · b62a07c
2 parents 37167d5 + e26d6b7
commit b62a07c
Show file tree

Hide file tree

Showing 15 changed files with 102 additions and 70 deletions.
diff --git a/changelog.html b/changelog.html
@@ -54,6 +54,11 @@
 
 <!-- Record your changes in the trunk here. -->
 <div id="trunk" style="display:none"><!--=TRUNK-BEGIN=-->
+<ul class=image>
+  <li class=>
+</ul>
+</div><!--=TRUNK-END=-->
+<h3><a name=v1.609>What's new in 1.609</a> (2015/04/12)</h3>
 <ul class=image>
   <li class=bug>
     When concurrent builds are enabled, artifact retention policy may delete artifact being 
@@ -62,9 +67,7 @@
   <li class=bug>
     Documentation for $BUILD_ID did not reflect current reality
     (<a href="https://issues.jenkins-ci.org/browse/JENKINS-26520">issue 26520</a>)
-  <li class=>
 </ul>
-</div><!--=TRUNK-END=-->
 <h3><a name=v1.608>What's new in 1.608</a> (2015/04/05)</h3>
 <ul class=image>
   <li class=bug>

diff --git a/cli/pom.xml b/cli/pom.xml
@@ -5,7 +5,7 @@
   <parent>
     <groupId>org.jenkins-ci.main</groupId>
     <artifactId>pom</artifactId>
-    <version>1.609-SNAPSHOT</version>
+    <version>1.610-SNAPSHOT</version>
   </parent>
 
   <artifactId>cli</artifactId>

diff --git a/core/pom.xml b/core/pom.xml
@@ -29,7 +29,7 @@ THE SOFTWARE.
   <parent>
     <groupId>org.jenkins-ci.main</groupId>
     <artifactId>pom</artifactId>
-    <version>1.609-SNAPSHOT</version>
+    <version>1.610-SNAPSHOT</version>
   </parent>
 
   <artifactId>jenkins-core</artifactId>
@@ -276,6 +276,11 @@ THE SOFTWARE.
       <artifactId>commons-beanutils</artifactId>
       <version>1.8.3</version>
     </dependency>
+    <dependency>
+        <groupId>org.apache.commons</groupId>
+        <artifactId>commons-compress</artifactId>
+        <version>1.9</version>
+    </dependency>
     <dependency>
       <groupId>javax.mail</groupId>
       <artifactId>mail</artifactId>

diff --git a/core/src/main/java/hudson/FilePath.java b/core/src/main/java/hudson/FilePath.java
@@ -33,7 +33,6 @@
 import hudson.model.Computer;
 import hudson.model.Item;
 import hudson.model.TaskListener;
-import hudson.org.apache.tools.tar.TarInputStream;
 import hudson.os.PosixAPI;
 import hudson.os.PosixException;
 import hudson.remoting.Callable;
@@ -70,7 +69,6 @@
 import org.apache.tools.ant.DirectoryScanner;
 import org.apache.tools.ant.Project;
 import org.apache.tools.ant.types.FileSet;
-import org.apache.tools.tar.TarEntry;
 import org.apache.tools.zip.ZipEntry;
 import org.apache.tools.zip.ZipFile;
 import org.kohsuke.stapler.Stapler;
@@ -120,6 +118,8 @@
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import jenkins.security.MasterToSlaveCallable;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
 import org.jenkinsci.remoting.RoleChecker;
 import org.jenkinsci.remoting.RoleSensitive;
 
@@ -2266,12 +2266,15 @@ private Integer writeToTar(File baseDir, DirScanner scanner, OutputStream out) t
 
     /**
      * Reads from a tar stream and stores obtained files to the base dir.
+     * @since TODO supports large files > 10 GB, migration to commons-compress
      */
     private void readFromTar(String name, File baseDir, InputStream in) throws IOException {
-        TarInputStream t = new TarInputStream(in);
+        TarArchiveInputStream t = new TarArchiveInputStream(in);
+
+        // TarInputStream t = new TarInputStream(in);
         try {
-            TarEntry te;
-            while ((te = t.getNextEntry()) != null) {
+            TarArchiveEntry te;
+            while ((te = t.getNextTarEntry()) != null) {
                 File f = new File(baseDir,te.getName());
                 if(te.isDirectory()) {
                     mkdirs(f);
@@ -2280,8 +2283,7 @@ private void readFromTar(String name, File baseDir, InputStream in) throws IOExc
                     if (parent != null) mkdirs(parent);
                     writing(f);
 
-                    byte linkFlag = (Byte) LINKFLAG_FIELD.get(te);
-                    if (linkFlag==TarEntry.LF_SYMLINK) {
+                    if (te.isSymbolicLink()) {
                         new FilePath(f).symlinkTo(te.getLinkName(), TaskListener.NULL);
                     } else {
                         IOUtils.copy(t,f);
@@ -2298,8 +2300,6 @@ private void readFromTar(String name, File baseDir, InputStream in) throws IOExc
         } catch (InterruptedException e) {
             Thread.currentThread().interrupt(); // process this later
             throw new IOException("Failed to extract "+name,e);
-        } catch (IllegalAccessException e) {
-            throw new IOException("Failed to extract "+name,e);
         } finally {
             t.close();
         }
@@ -2722,20 +2722,6 @@ public int compare(String o1, String o2) {
         }
     };
 
-    private static final Field LINKFLAG_FIELD = getTarEntryLinkFlagField();
-
-    private static Field getTarEntryLinkFlagField() {
-        try {
-            Field f = TarEntry.class.getDeclaredField("linkFlag");
-            f.setAccessible(true);
-            return f;
-        } catch (SecurityException e) {
-            throw new AssertionError(e);
-        } catch (NoSuchFieldException e) {
-            throw new AssertionError(e);
-        }
-    }
-
     /**
      * Gets the {@link FilePath} representation of the "~" directory
      * (User's home directory in the Unix sense) of the given channel.

diff --git a/core/src/main/java/hudson/model/Descriptor.java b/core/src/main/java/hudson/model/Descriptor.java
@@ -933,7 +933,7 @@ List<T> newInstancesFromHeteroList(StaplerRequest req, Object formData,
     /**
      * Finds a descriptor from a collection by its ID.
      * @param id should match {@link #getId}
-     * @since TODO
+     * @since 1.610
      */
     public static @CheckForNull <T extends Descriptor> T findById(Collection<? extends T> list, String id) {
         for (T d : list) {
@@ -958,7 +958,7 @@ List<T> newInstancesFromHeteroList(StaplerRequest req, Object formData,
     /**
      * Finds a descriptor from a collection by the class name of the {@link Describable} it describes.
      * @param className should match {@link Class#getName} of a {@link #clazz}
-     * @since TODO
+     * @since 1.610
      */
     public static @CheckForNull <T extends Descriptor> T findByDescribableClassName(Collection<? extends T> list, String className) {
         for (T d : list) {

diff --git a/core/src/main/java/hudson/org/apache/tools/tar/TarInputStream.java b/core/src/main/java/hudson/org/apache/tools/tar/TarInputStream.java
@@ -37,8 +37,9 @@
  * methods are provided to position at each successive entry in
  * the archive, and the read each entry as a normal input stream
  * using read().
- *
+ * @deprecated Use {@link org.apache.commons.compress.archivers.tar.TarArchiveInputStream} instead
  */
+@Deprecated
 public class TarInputStream extends FilterInputStream {
 
     // CheckStyle:VisibilityModifier OFF - bc

diff --git a/core/src/main/java/hudson/org/apache/tools/tar/TarOutputStream.java b/core/src/main/java/hudson/org/apache/tools/tar/TarOutputStream.java
@@ -35,8 +35,11 @@
  * The TarOutputStream writes a UNIX tar archive as an OutputStream.
  * Methods are provided to put entries, and then write their contents
  * by writing to this stream using write().
+ * 
+ * @deprecated Use {@link org.apache.commons.compress.archivers.tar.TarArchiveOutputStream} instead
  *
  */
+@Deprecated
 public class TarOutputStream extends FilterOutputStream {
     /** Fail if a long file name is required in the archive. */
     public static final int LONGFILE_ERROR = 0;

diff --git a/core/src/main/java/hudson/tools/DownloadFromUrlInstaller.java b/core/src/main/java/hudson/tools/DownloadFromUrlInstaller.java
@@ -129,6 +129,7 @@ protected Downloadable createDownloadable() {
          * This ID needs to be unique, and needs to match the ID token in the JSON update file.
          * <p>
          * By default we use the fully-qualified class name of the {@link DownloadFromUrlInstaller} subtype.
+         * @since 1.610
          */
         public String getDownloadableId() {
             return clazz.getName().replace('$','.');

diff --git a/core/src/main/java/hudson/util/io/TarArchiver.java b/core/src/main/java/hudson/util/io/TarArchiver.java
@@ -37,6 +37,8 @@
 import java.io.IOException;
 import java.io.OutputStream;
 import java.lang.reflect.Field;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
 
 import static org.apache.tools.tar.TarConstants.LF_SYMLINK;
 
@@ -47,24 +49,25 @@
  */
 final class TarArchiver extends Archiver {
     private final byte[] buf = new byte[8192];
-    private final TarOutputStream tar;
+    private final TarArchiveOutputStream tar;
 
     TarArchiver(OutputStream out) {
-        tar = new TarOutputStream(new BufferedOutputStream(out) {
+        tar = new TarArchiveOutputStream(new BufferedOutputStream(out) {
             // TarOutputStream uses TarBuffer internally,
             // which flushes the stream for each block. this creates unnecessary
             // data stream fragmentation, and flush request to a remote, which slows things down.
             @Override
             public void flush() throws IOException {
                 // so don't do anything in flush
             }
-        });
-        tar.setLongFileMode(TarOutputStream.LONGFILE_GNU);
+        });    
+        tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR);
+        tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_GNU);
     }
 
     @Override
     public void visitSymlink(File link, String target, String relativePath) throws IOException {
-        TarEntry e = new TarEntry(relativePath, LF_SYMLINK);
+        TarArchiveEntry e = new TarArchiveEntry(relativePath, LF_SYMLINK);
         try {
             int mode = IOUtils.mode(link);
             if (mode != -1) {
@@ -73,16 +76,11 @@ public void visitSymlink(File link, String target, String relativePath) throws I
         } catch (PosixException x) {
             // ignore
         }
+
+        e.setLinkName(target);
 
-        try {
-            StringBuffer linkName = (StringBuffer) LINKNAME_FIELD.get(e);
-            linkName.setLength(0);
-            linkName.append(target);
-        } catch (IllegalAccessException x) {
-            throw new IOException("Failed to set linkName", x);
-        }
-
-        tar.putNextEntry(e);
+        tar.putArchiveEntry(e);
+        tar.closeArchiveEntry();
         entriesWritten++;
     }
 
@@ -97,14 +95,14 @@ public void visit(File file, String relativePath) throws IOException {
 
         if(file.isDirectory())
             relativePath+='/';
-        TarEntry te = new TarEntry(relativePath);
+        TarArchiveEntry te = new TarArchiveEntry(relativePath);
         int mode = IOUtils.mode(file);
         if (mode!=-1)   te.setMode(mode);
         te.setModTime(file.lastModified());
         if(!file.isDirectory())
             te.setSize(file.length());
 
-        tar.putNextEntry(te);
+        tar.putArchiveEntry(te);
 
         if (!file.isDirectory()) {
             FileInputStream in = new FileInputStream(file);
@@ -117,25 +115,11 @@ public void visit(File file, String relativePath) throws IOException {
             }
         }
 
-        tar.closeEntry();
+        tar.closeArchiveEntry();
         entriesWritten++;
     }
 
     public void close() throws IOException {
         tar.close();
     }
-
-    private static final Field LINKNAME_FIELD = getTarEntryLinkNameField();
-
-    private static Field getTarEntryLinkNameField() {
-        try {
-            Field f = TarEntry.class.getDeclaredField("linkName");
-            f.setAccessible(true);
-            return f;
-        } catch (SecurityException e) {
-            throw new AssertionError(e);
-        } catch (NoSuchFieldException e) {
-            throw new AssertionError(e);
-        }
-    }
 }
diff --git a/core/src/main/java/jenkins/AgentProtocol.java b/core/src/main/java/jenkins/AgentProtocol.java
@@ -4,7 +4,6 @@
 import hudson.ExtensionList;
 import hudson.ExtensionPoint;
 import hudson.TcpSlaveAgentListener;
-import hudson.model.AperiodicWork;
 import jenkins.model.Jenkins;
 
 import java.io.IOException;
@@ -40,7 +39,7 @@ public abstract class AgentProtocol implements ExtensionPoint {
     public abstract void handle(Socket socket) throws IOException, InterruptedException;
 
     /**
-     * Returns all the registered {@link AperiodicWork}s.
+     * Returns all the registered {@link AgentProtocol}s.
      */
     public static ExtensionList<AgentProtocol> all() {
         return ExtensionList.lookup(AgentProtocol.class);

diff --git a/core/src/test/java/hudson/FilePathTest.java b/core/src/test/java/hudson/FilePathTest.java
@@ -35,6 +35,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.io.RandomAccessFile;
 import java.net.ConnectException;
 import java.net.HttpURLConnection;
 import java.net.URL;
@@ -56,6 +57,8 @@
 import org.apache.tools.ant.Project;
 import org.apache.tools.ant.taskdefs.Chmod;
 import static org.junit.Assert.*;
+import static org.junit.Assume.assumeTrue;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
@@ -325,6 +328,53 @@ private FilePath createFilePath(final File base, final String... path) throws IO
         FileUtils.touch(building);
         return new FilePath(building);
     }
+
+    /**
+     * Performs round-trip archiving for Tar handling methods.
+     * @throws Exception test failure
+     */
+    @Test public void compressTarUntarRoundTrip() throws Exception {
+        checkTarUntarRoundTrip("compressTarUntarRoundTrip_zero", 0);   
+        checkTarUntarRoundTrip("compressTarUntarRoundTrip_small", 100); 
+        checkTarUntarRoundTrip("compressTarUntarRoundTrip_medium", 50000); 
+    }
+
+    /**
+     * Checks that big files (>8GB) can be archived and then unpacked.
+     * This test is disabled by default due the impact on RAM.
+     * The actual file size limit is 8589934591 bytes.
+     * @throws Exception test failure
+     */
+    @Issue("JENKINS-10629")
+    @Ignore
+    @Test public void archiveBigFile() throws Exception {
+        final long largeFileSize = 9000000000L; // >8589934591 bytes
+        final String filePrefix = "JENKINS-10629";
+        checkTarUntarRoundTrip(filePrefix, largeFileSize);
+    }
+
+    private void checkTarUntarRoundTrip(String filePrefix, long fileSize) throws Exception {
+        final File tmpDir = temp.newFolder(filePrefix);
+        final File tempFile =  new File(tmpDir, filePrefix + ".log");
+        RandomAccessFile file = new RandomAccessFile(tempFile, "rw");
+        final File tarFile = new File(tmpDir, filePrefix + ".tar");
+
+        file.setLength(fileSize);
+        assumeTrue(fileSize == file.length());
+        file.close();
+
+        // Compress archive
+        final FilePath tmpDirPath = new FilePath(tmpDir);
+        int tar = tmpDirPath.tar(new FileOutputStream(tarFile), tempFile.getName());
+        assertEquals("One file should have been compressed", 1, tar);
+
+        // Decompress
+        FilePath outDir = new FilePath(temp.newFolder(filePrefix + "_out"));
+        final FilePath outFile = outDir.child(tempFile.getName());
+        tmpDirPath.child( filePrefix + ".tar").untar(outDir, TarCompression.NONE);
+        assertEquals("Result file after the roundtrip differs from the initial file",
+                new FilePath(tempFile).digest(), outFile.digest());
+    }
 
     @Test public void list() throws Exception {
         File baseDir = temp.getRoot();