Navigation Menu

Skip to content

Commit

Permalink
Merge branch 'master' into JENKINS-50726
Browse files Browse the repository at this point in the history
  • Loading branch information
@carlossg
carlossg committed May 30, 2018
2 parents 5df774d + b09776a commit 7645356
Show file tree
Hide file tree
Showing 8 changed files with 219 additions and 69 deletions.
17 changes: 14 additions & 3 deletions pom.xml
View file
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>plugin</artifactId>
<version>3.10</version>
<version>3.11</version>
<relativePath />
</parent>
<groupId>io.jenkins.plugins</groupId>
Expand All @@ -19,7 +19,7 @@
<jclouds.version>2.0.3</jclouds.version>
<jenkins.version>2.121</jenkins.version>
<java.level>8</java.level>
<workflow-api-plugin.version>2.28-rc337.8abe7c5204d9</workflow-api-plugin.version> <!-- TODO https://github.com/jenkinsci/workflow-api-plugin/pull/67 -->
<workflow-api-plugin.version>2.28-rc341.90cc5dc659de</workflow-api-plugin.version> <!-- TODO https://github.com/jenkinsci/workflow-api-plugin/pull/67 -->
<useBeta>true</useBeta>
</properties>

Expand Down Expand Up @@ -67,7 +67,7 @@
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.11.248</version>
<version>1.11.329</version>
</dependency>
<!--
<dependency>
Expand Down Expand Up @@ -159,6 +159,17 @@
<version>1.7</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.rholder</groupId>
<artifactId>guava-retrying</artifactId>
<version>2.0.0</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
Expand Down
6 changes: 6 additions & 0 deletions src/main/java/io/jenkins/plugins/artifact_manager_jclouds/BlobStoreProvider.java
View file
Expand Up @@ -59,6 +59,12 @@ public enum HttpMethod {
@NonNull
public abstract String getContainer();

/** A constant to define whether we should delete blobs or leave them to be managed on the blob service side. */
public abstract boolean isDeleteBlobs();

/** A constant to define whether we should delete stashes or leave them to be managed on the blob service side. */
public abstract boolean isDeleteStashes();

/** Creates the jclouds handle for working with blobs. */
@NonNull
public abstract BlobStoreContext getContext() throws IOException;
Expand Down
100 changes: 76 additions & 24 deletions src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java
View file
Expand Up @@ -24,6 +24,7 @@

package io.jenkins.plugins.artifact_manager_jclouds;

import com.github.rholder.retry.Attempt;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -37,6 +38,7 @@
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.logging.Level;
import java.util.logging.Logger;

Expand All @@ -48,6 +50,11 @@
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jenkinsci.plugins.workflow.flow.StashManager;

import com.github.rholder.retry.RetryException;
import com.github.rholder.retry.RetryListener;
import com.github.rholder.retry.RetryerBuilder;
import com.github.rholder.retry.StopStrategies;
import com.github.rholder.retry.WaitStrategies;
import hudson.AbortException;
import hudson.EnvVars;
import hudson.FilePath;
Expand All @@ -61,10 +68,12 @@
import hudson.util.DirScanner;
import hudson.util.io.ArchiverFactory;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider.HttpMethod;
import java.util.concurrent.TimeUnit;
import jenkins.MasterToSlaveFileCallable;
import jenkins.model.ArtifactManager;
import jenkins.util.VirtualFile;
import org.apache.commons.io.IOUtils;
import org.apache.http.client.HttpResponseException;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;

Expand Down Expand Up @@ -124,13 +133,18 @@ public void archive(FilePath workspace, Launcher launcher, BuildListener listene
artifactUrls.put(entry.getValue(), provider.toExternalURL(blob, HttpMethod.PUT));
}

workspace.act(new UploadToBlobStorage(artifactUrls));
workspace.act(new UploadToBlobStorage(artifactUrls, listener));
listener.getLogger().printf("Uploaded %s artifact(s) to %s%n", artifactUrls.size(), provider.toURI(provider.getContainer(), getBlobPath("artifacts/")));
}

@Override
public boolean delete() throws IOException, InterruptedException {
return delete(provider, getContext().getBlobStore(), getBlobPath(""));
String blobPath = getBlobPath("");
if (!provider.isDeleteBlobs()) {
LOGGER.log(Level.FINE, "Ignoring blob deletion: {0}", blobPath);
return false;
}
return delete(provider, getContext().getBlobStore(), blobPath);
}

/**
Expand Down Expand Up @@ -168,7 +182,7 @@ public void stash(String name, FilePath workspace, Launcher launcher, EnvVars en
Blob blob = blobStore.blobBuilder(path).build();
blob.getMetadata().setContainer(provider.getContainer());
URL url = provider.toExternalURL(blob, HttpMethod.PUT);
int count = workspace.act(new Stash(url, includes, excludes, useDefaultExcludes, WorkspaceList.tempDir(workspace).getRemote()));
int count = workspace.act(new Stash(url, includes, excludes, useDefaultExcludes, WorkspaceList.tempDir(workspace).getRemote(), listener));
if (count == 0 && !allowEmpty) {
throw new AbortException("No files included in stash");
}
Expand All @@ -181,13 +195,15 @@ private static final class Stash extends MasterToSlaveFileCallable<Integer> {
private final String includes, excludes;
private final boolean useDefaultExcludes;
private final String tempDir;
private final TaskListener listener;

Stash(URL url, String includes, String excludes, boolean useDefaultExcludes, String tempDir) throws IOException {
Stash(URL url, String includes, String excludes, boolean useDefaultExcludes, String tempDir, TaskListener listener) throws IOException {
this.url = url;
this.includes = includes;
this.excludes = excludes;
this.useDefaultExcludes = useDefaultExcludes;
this.tempDir = tempDir;
this.listener = listener;
}

@Override
Expand All @@ -205,7 +221,7 @@ public Integer invoke(File f, VirtualChannel channel) throws IOException, Interr
throw new IOException(e);
}
if (count > 0) {
uploadFile(tmp, url);
uploadFile(tmp, url, listener);
}
return count;
} finally {
Expand Down Expand Up @@ -251,6 +267,12 @@ public Void invoke(File f, VirtualChannel channel) throws IOException, Interrupt
@Override
public void clearAllStashes(TaskListener listener) throws IOException, InterruptedException {
String stashPrefix = getBlobPath("stashes/");

if (!provider.isDeleteStashes()) {
LOGGER.log(Level.FINE, "Ignoring stash deletion: {0}", stashPrefix);
return;
}

BlobStore blobStore = getContext().getBlobStore();
int count = 0;
try {
Expand Down Expand Up @@ -304,19 +326,19 @@ private static class UploadToBlobStorage extends MasterToSlaveFileCallable<Void>
private static final long serialVersionUID = 1L;

private final Map<String, URL> artifactUrls; // e.g. "target/x.war", "http://..."
private final TaskListener listener;

UploadToBlobStorage(Map<String, URL> artifactUrls) {
UploadToBlobStorage(Map<String, URL> artifactUrls, TaskListener listener) {
this.artifactUrls = artifactUrls;
this.listener = listener;
}

@Override
public Void invoke(File f, VirtualChannel channel) throws IOException, InterruptedException {
for (Map.Entry<String, URL> entry : artifactUrls.entrySet()) {
Path local = f.toPath().resolve(entry.getKey());
URL url = entry.getValue();
LOGGER.log(Level.FINE, "Uploading {0} to {1}",
new String[] { local.toAbsolutePath().toString(), url.toString() });
uploadFile(local, url);
uploadFile(local, url, listener);
}
return null;
}
Expand All @@ -325,23 +347,53 @@ public Void invoke(File f, VirtualChannel channel) throws IOException, Interrupt
/**
* Upload a file to a URL
*/
private static void uploadFile(Path f, URL url) throws IOException {
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true);
connection.setRequestMethod("PUT");
connection.setFixedLengthStreamingMode(Files.size(f)); // prevent loading file in memory
try (OutputStream out = connection.getOutputStream()) {
Files.copy(f, out);
}
int responseCode = connection.getResponseCode();
@SuppressWarnings("Convert2Lambda") // bogus use of generics (type variable should have been on class); cannot be made into a lambda
private static void uploadFile(Path f, URL url, final TaskListener listener) throws IOException {
String urlSafe = url.toString().replaceFirst("[?].+$", "?…");
if (responseCode < 200 || responseCode >= 300) {
String diag;
try (InputStream err = connection.getErrorStream()) {
diag = err != null ? IOUtils.toString(err, connection.getContentEncoding()) : null;
try {
RetryerBuilder.<Void>newBuilder().
retryIfException(x -> x instanceof IOException && (!(x instanceof HttpResponseException) || ((HttpResponseException) x).getStatusCode() >= 500)).
withRetryListener(new RetryListener() {
@Override
public <Void> void onRetry(Attempt<Void> attempt) {
if (attempt.hasException()) {
listener.getLogger().println("Retrying upload after: " + attempt.getExceptionCause());
}
}
}).
// TODO all scalars configurable via system property
withStopStrategy(StopStrategies.stopAfterAttempt(10)).
// Note that this is not a _randomized_ exponential backoff; and the base of the exponent is hard-coded to 2.
withWaitStrategy(WaitStrategies.exponentialWait(100, 5, TimeUnit.MINUTES)).
// TODO withAttemptTimeLimiter(…).
build().call(() -> {
HttpURLConnection connection = (HttpURLConnection) url.openConnection();
connection.setDoOutput(true);
connection.setRequestMethod("PUT");
connection.setFixedLengthStreamingMode(Files.size(f)); // prevent loading file in memory
try (OutputStream out = connection.getOutputStream()) {
Files.copy(f, out);
}
int responseCode = connection.getResponseCode();
if (responseCode < 200 || responseCode >= 300) {
String diag;
try (InputStream err = connection.getErrorStream()) {
diag = err != null ? IOUtils.toString(err, connection.getContentEncoding()) : null;
}
throw new HttpResponseException(responseCode, String.format("Failed to upload %s to %s, response: %d %s, body: %s", f.toAbsolutePath(), urlSafe, responseCode, connection.getResponseMessage(), diag));
}
return null;
});
} catch (ExecutionException | RetryException x) { // *sigh*, checked exceptions
Throwable x2 = x.getCause();
if (x2 instanceof IOException) {
throw (IOException) x2;
} else if (x2 instanceof RuntimeException) {
throw (RuntimeException) x2;
} else { // Error?
throw new RuntimeException(x);
}
throw new IOException(String.format("Failed to upload %s to %s, response: %d %s, body: %s", f.toAbsolutePath(), urlSafe, responseCode, connection.getResponseMessage(), diag));
}
LOGGER.log(Level.FINE, "Uploaded {0} to {1}: {2}", new Object[] { f.toAbsolutePath(), urlSafe, responseCode });
}

}
29 changes: 22 additions & 7 deletions src/main/java/io/jenkins/plugins/artifact_manager_s3/S3BlobStore.java
View file
Expand Up @@ -24,8 +24,6 @@

package io.jenkins.plugins.artifact_manager_s3;

import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProviderDescriptor;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
Expand All @@ -41,6 +39,10 @@

import javax.annotation.Nonnull;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSSessionCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;

import org.apache.commons.lang.StringUtils;
import org.jclouds.ContextBuilder;
import org.jclouds.aws.domain.Region;
Expand All @@ -54,15 +56,13 @@
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.DataBoundConstructor;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSSessionCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import org.kohsuke.stapler.DataBoundSetter;

import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import hudson.util.ListBoxModel;
import org.kohsuke.stapler.DataBoundSetter;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProviderDescriptor;
import shaded.com.google.common.base.Supplier;

/**
Expand All @@ -84,6 +84,11 @@ public class S3BlobStore extends BlobStoreProvider {
public final String CONTAINER_PROPERTY = System.getProperty(KEY_CONTAINER);
public final String REGION_PROPERTY = System.getProperty(KEY_REGION);

@SuppressWarnings("FieldMayBeFinal")
private static boolean DELETE_BLOBS = Boolean.getBoolean(S3BlobStore.class.getName() + ".deleteBlobs");
@SuppressWarnings("FieldMayBeFinal")
private static boolean DELETE_STASHES = Boolean.getBoolean(S3BlobStore.class.getName() + ".deleteStashes");

private String container;
private String prefix;
private String region;
Expand Down Expand Up @@ -121,6 +126,16 @@ public void setRegion(String region) {
this.region = region;
}

@Override
public boolean isDeleteBlobs() {
return DELETE_BLOBS;
}

@Override
public boolean isDeleteStashes() {
return DELETE_STASHES;
}

@Override
public BlobStoreContext getContext() throws IOException {
LOGGER.log(Level.FINEST, "Building context");
Expand Down

0 comments on commit 7645356

Please sign in to comment.