Skip to content

Commit

Permalink
Merge branch 'master' into JENKINS-51523
Browse files Browse the repository at this point in the history
  • Loading branch information
@carlossg
carlossg committed May 24, 2018
2 parents 11f2455 + 333330c commit 479c7ce
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 39 deletions.
6 changes: 6 additions & 0 deletions README.md
View file
Expand Up @@ -128,3 +128,9 @@ docker run --rm -i --init -v <YOUR_HOME_DIR>/.aws:/home/jenkins/.aws -e AWS_REGI
```

but note that the mount will not work if your `~/.aws/` is set to be readable only by the user.


# Force the Region

If you have problems detecting the region on your environment you can force the region setting by adding this property
`-Dio.jenkins.plugins.artifact_manager_s3.S3BlobStore.region=REGION_NAME` to the Jenkins JVM options.
53 changes: 44 additions & 9 deletions pom.xml
View file
Expand Up @@ -17,9 +17,9 @@
<revision>1.0-alpha-1</revision>
<changelist>-SNAPSHOT</changelist>
<jclouds.version>2.0.3</jclouds.version>
<jenkins.version>2.118</jenkins.version>
<jenkins.version>2.121</jenkins.version>
<java.level>8</java.level>
<workflow-api-plugin.version>2.28-rc333.0675e9e9cb4c</workflow-api-plugin.version> <!-- TODO https://github.com/jenkinsci/workflow-api-plugin/pull/67 -->
<workflow-api-plugin.version>2.28-rc337.8abe7c5204d9</workflow-api-plugin.version> <!-- TODO https://github.com/jenkinsci/workflow-api-plugin/pull/67 -->
<useBeta>true</useBeta>
</properties>

Expand Down Expand Up @@ -63,13 +63,6 @@
-->
</exclusions>
</dependency>
<!-- to make it work in eclipse during development -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>16.0.1</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
Expand Down Expand Up @@ -136,6 +129,30 @@
<version>1.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
<artifactId>workflow-job</artifactId>
<version>2.21</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
<artifactId>workflow-cps</artifactId>
<version>2.53</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
<artifactId>workflow-durable-task-step</artifactId>
<version>2.19</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
<artifactId>workflow-basic-steps</artifactId>
<version>2.8-rc351.c6608322f479</version> <!-- TODO https://github.com/jenkinsci/workflow-basic-steps-plugin/pull/60 -->
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
Expand All @@ -154,6 +171,24 @@
<artifactId>guice</artifactId>
<version>3.0</version>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
<artifactId>workflow-step-api</artifactId>
<version>2.15</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins.workflow</groupId>
<artifactId>workflow-support</artifactId>
<version>2.19-rc289.691fb60f3fa6</version> <!-- TODO https://github.com/jenkinsci/workflow-support-plugin/pull/61 -->
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.jenkins-ci.plugins</groupId>
<artifactId>scm-api</artifactId>
<version>2.2.7</version>
<scope>test</scope>
</dependency>
</dependencies>
</dependencyManagement>

Expand Down
66 changes: 39 additions & 27 deletions src/main/java/io/jenkins/plugins/artifact_manager_jclouds/JCloudsArtifactManager.java
View file
Expand Up @@ -141,17 +141,21 @@ public boolean delete() throws IOException, InterruptedException {
* Delete all blobs starting with prefix
*/
public static boolean delete(BlobStoreProvider provider, BlobStore blobStore, String prefix) throws IOException, InterruptedException {
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(prefix).recursive());
boolean found = false;
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(prefix);
LOGGER.fine("deleting " + path);
blobStore.removeBlob(provider.getContainer(), path);
found = true;
try {
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(prefix).recursive());
boolean found = false;
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(prefix);
LOGGER.fine("deleting " + path);
blobStore.removeBlob(provider.getContainer(), path);
found = true;
}
return found;
} catch (RuntimeException x) {
throw new IOException(x);
}
return found;
}

@Override
Expand Down Expand Up @@ -258,15 +262,19 @@ public void clearAllStashes(TaskListener listener) throws IOException, Interrupt
}

BlobStore blobStore = getContext().getBlobStore();
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(stashPrefix).recursive());
int count = 0;
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(stashPrefix);
LOGGER.fine("deleting " + path);
blobStore.removeBlob(provider.getContainer(), path);
count++;
try {
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(stashPrefix).recursive());
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(stashPrefix);
LOGGER.fine("deleting " + path);
blobStore.removeBlob(provider.getContainer(), path);
count++;
}
} catch (RuntimeException x) {
throw new IOException(x);
}
listener.getLogger().printf("Deleted %d stash(es) from %s%n", count, provider.toURI(provider.getContainer(), stashPrefix));
}
Expand All @@ -280,16 +288,20 @@ public void copyAllArtifactsAndStashes(Run<?, ?> to, TaskListener listener) thro
JCloudsArtifactManager dest = (JCloudsArtifactManager) am;
String allPrefix = getBlobPath("");
BlobStore blobStore = getContext().getBlobStore();
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(allPrefix).recursive());
int count = 0;
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(allPrefix);
String destPath = getBlobPath(dest.key, path.substring(allPrefix.length()));
LOGGER.fine("copying " + path + " to " + destPath);
blobStore.copyBlob(provider.getContainer(), path, provider.getContainer(), destPath, CopyOptions.NONE);
count++;
try {
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(allPrefix).recursive());
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(allPrefix);
String destPath = getBlobPath(dest.key, path.substring(allPrefix.length()));
LOGGER.fine("copying " + path + " to " + destPath);
blobStore.copyBlob(provider.getContainer(), path, provider.getContainer(), destPath, CopyOptions.NONE);
count++;
}
} catch (RuntimeException x) {
throw new IOException(x);
}
listener.getLogger().printf("Copied %d artifact(s)/stash(es) from %s to %s%n", count, provider.toURI(provider.getContainer(), allPrefix), provider.toURI(provider.getContainer(), dest.getBlobPath("")));
}
Expand Down
31 changes: 28 additions & 3 deletions src/main/java/io/jenkins/plugins/artifact_manager_s3/S3BlobStore.java
View file
Expand Up @@ -32,23 +32,29 @@
import java.net.URLEncoder;
import java.util.Date;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;

import javax.annotation.Nonnull;
import javax.ws.rs.HEAD;

import io.jenkins.plugins.artifact_manager_jclouds.JCloudsArtifactManager;
import org.apache.commons.lang.StringUtils;
import org.jclouds.ContextBuilder;
import org.jclouds.aws.domain.SessionCredentials;
import org.jclouds.aws.s3.AWSS3ProviderMetadata;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.domain.Credentials;
import org.jclouds.location.reference.LocationConstants;
import org.jclouds.osgi.ProviderRegistry;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.DataBoundConstructor;

import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSSessionCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;

Expand All @@ -75,6 +81,8 @@ public class S3BlobStore extends BlobStoreProvider {
@SuppressWarnings("FieldMayBeFinal")
private static String PREFIX = System.getenv("S3_DIR");
@SuppressWarnings("FieldMayBeFinal")
private static String REGION = System.getProperty(S3BlobStore.class.getName() + ".region");
@SuppressWarnings("FieldMayBeFinal")
private static boolean DELETE_BLOBS = Boolean.getBoolean(S3BlobStore.class.getName() + ".deleteBlobs");
@SuppressWarnings("FieldMayBeFinal")
private static boolean DELETE_STASHES = Boolean.getBoolean(S3BlobStore.class.getName() + ".deleteStashes");
Expand Down Expand Up @@ -107,26 +115,43 @@ public BlobStoreContext getContext() throws IOException {
LOGGER.log(Level.FINEST, "Building context");
ProviderRegistry.registerProvider(AWSS3ProviderMetadata.builder().build());
try {
Properties props = new Properties();

if(StringUtils.isNotBlank(REGION)) {
props.setProperty(LocationConstants.PROPERTY_REGIONS, REGION);
}

return ContextBuilder.newBuilder("aws-s3").credentialsSupplier(getCredentialsSupplier())
.buildView(BlobStoreContext.class);
} catch (NoSuchElementException x) {
throw new IOException(x);
}
}

static boolean BREAK_CREDS;

private Supplier<Credentials> getCredentialsSupplier() throws IOException {
// get user credentials from env vars, profiles,...
AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard();
// Assume we are using session credentials
AWSSessionCredentials awsCredentials = (AWSSessionCredentials) builder.getCredentials().getCredentials();
AWSCredentials awsCredentials = builder.getCredentials().getCredentials();
if (awsCredentials == null) {
throw new IOException("Unable to get credentials from environment");
}

// Assume we are using session credentials
if(!(awsCredentials instanceof AWSSessionCredentials)){
throw new IOException("No valid session credentials");
}

String sessionToken = ((AWSSessionCredentials) awsCredentials).getSessionToken();
if (BREAK_CREDS) {
sessionToken = "<broken>";
}

SessionCredentials sessionCredentials = SessionCredentials.builder()
.accessKeyId(awsCredentials.getAWSAccessKeyId()) //
.secretAccessKey(awsCredentials.getAWSSecretKey()) //
.sessionToken(awsCredentials.getSessionToken()) //
.sessionToken(sessionToken) //
.build();

return () -> sessionCredentials;
Expand Down
20 changes: 20 additions & 0 deletions src/test/java/io/jenkins/plugins/artifact_manager_s3/JCloudsArtifactManagerTest.java
View file
Expand Up @@ -69,6 +69,10 @@
import jenkins.model.ArtifactManagerFactory;
import jenkins.model.Jenkins;
import jenkins.security.MasterToSlaveCallable;
import org.jenkinsci.plugins.workflow.cps.CpsFlowDefinition;
import org.jenkinsci.plugins.workflow.job.WorkflowJob;
import org.jenkinsci.plugins.workflow.job.WorkflowRun;
import org.jvnet.hudson.test.Issue;
import org.jclouds.blobstore.BlobStoreContext;
import org.jclouds.blobstore.domain.Blob;

Expand Down Expand Up @@ -194,6 +198,22 @@ public boolean perform(AbstractBuild<?, ?> build, Launcher launcher, BuildListen
assertThat(httpCount, lessThanOrEqualTo(11));
}

@Issue({"JENKINS-51390", "JCLOUDS-1200"})
@Test
public void serializationProblem() throws Exception {
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory());
WorkflowJob p = j.createProject(WorkflowJob.class, "p");
p.setDefinition(new CpsFlowDefinition("node {writeFile file: 'f', text: 'content'; archiveArtifacts 'f'; dir('d') {try {unarchive mapping: ['f': 'f']} catch (x) {sleep 1; echo(/caught $x/)}}}", true));
S3BlobStore.BREAK_CREDS = true;
try {
WorkflowRun b = j.buildAndAssertSuccess(p);
j.assertLogContains("caught org.jclouds.aws.AWSResponseException", b);
j.assertLogNotContains("java.io.NotSerializableException", b);
} finally {
S3BlobStore.BREAK_CREDS = false;
}
}

//@Test
public void archiveSingleLargeFile() throws Exception {
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory());
Expand Down

0 comments on commit 479c7ce

Please sign in to comment.