Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #39 from jenkinsci/no-delete-JENKINS-51523
[JENKINS-51523] Delegate artifact lifecycle management to S3 (redux)
  • Loading branch information
carlossg committed May 30, 2018
2 parents 7407cbd + bc31751 commit b09776a
Show file tree
Hide file tree
Showing 7 changed files with 98 additions and 25 deletions.
2 changes: 1 addition & 1 deletion pom.xml
Expand Up @@ -19,7 +19,7 @@
<jclouds.version>2.0.3</jclouds.version>
<jenkins.version>2.121</jenkins.version>
<java.level>8</java.level>
<workflow-api-plugin.version>2.28-rc337.8abe7c5204d9</workflow-api-plugin.version> <!-- TODO https://github.com/jenkinsci/workflow-api-plugin/pull/67 -->
<workflow-api-plugin.version>2.28-rc341.90cc5dc659de</workflow-api-plugin.version> <!-- TODO https://github.com/jenkinsci/workflow-api-plugin/pull/67 -->
<useBeta>true</useBeta>
</properties>

Expand Down
Expand Up @@ -59,6 +59,12 @@ public enum HttpMethod {
@NonNull
public abstract String getContainer();

/** A constant to define whether we should delete blobs or leave them to be managed on the blob service side. */
public abstract boolean isDeleteBlobs();

/** A constant to define whether we should delete stashes or leave them to be managed on the blob service side. */
public abstract boolean isDeleteStashes();

/** Creates the jclouds handle for working with blobs. */
@NonNull
public abstract BlobStoreContext getContext() throws IOException;
Expand Down
Expand Up @@ -139,7 +139,12 @@ public void archive(FilePath workspace, Launcher launcher, BuildListener listene

@Override
public boolean delete() throws IOException, InterruptedException {
return delete(provider, getContext().getBlobStore(), getBlobPath(""));
String blobPath = getBlobPath("");
if (!provider.isDeleteBlobs()) {
LOGGER.log(Level.FINE, "Ignoring blob deletion: {0}", blobPath);
return false;
}
return delete(provider, getContext().getBlobStore(), blobPath);
}

/**
Expand Down Expand Up @@ -262,6 +267,12 @@ public Void invoke(File f, VirtualChannel channel) throws IOException, Interrupt
@Override
public void clearAllStashes(TaskListener listener) throws IOException, InterruptedException {
String stashPrefix = getBlobPath("stashes/");

if (!provider.isDeleteStashes()) {
LOGGER.log(Level.FINE, "Ignoring stash deletion: {0}", stashPrefix);
return;
}

BlobStore blobStore = getContext().getBlobStore();
int count = 0;
try {
Expand Down
Expand Up @@ -24,8 +24,6 @@

package io.jenkins.plugins.artifact_manager_s3;

import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProviderDescriptor;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URI;
Expand Down Expand Up @@ -62,6 +60,8 @@

import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.Extension;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProvider;
import io.jenkins.plugins.artifact_manager_jclouds.BlobStoreProviderDescriptor;
import shaded.com.google.common.base.Supplier;

/**
Expand All @@ -82,6 +82,10 @@ public class S3BlobStore extends BlobStoreProvider {
private static String PREFIX = System.getenv("S3_DIR");
@SuppressWarnings("FieldMayBeFinal")
private static String REGION = System.getProperty(S3BlobStore.class.getName() + ".region");
@SuppressWarnings("FieldMayBeFinal")
private static boolean DELETE_BLOBS = Boolean.getBoolean(S3BlobStore.class.getName() + ".deleteBlobs");
@SuppressWarnings("FieldMayBeFinal")
private static boolean DELETE_STASHES = Boolean.getBoolean(S3BlobStore.class.getName() + ".deleteStashes");

@DataBoundConstructor
public S3BlobStore() {}
Expand All @@ -96,6 +100,16 @@ public String getContainer() {
return BLOB_CONTAINER;
}

@Override
public boolean isDeleteBlobs() {
return DELETE_BLOBS;
}

@Override
public boolean isDeleteStashes() {
return DELETE_STASHES;
}

@Override
public BlobStoreContext getContext() throws IOException {
LOGGER.log(Level.FINEST, "Building context");
Expand Down
Expand Up @@ -155,4 +155,14 @@ public URL toExternalURL(Blob blob, HttpMethod httpMethod) throws IOException {
return new URL(baseURL, blob.getMetadata().getContainer() + "/" + blob.getMetadata().getName() + "?method=" + httpMethod);
}

@Override
public boolean isDeleteBlobs() {
return true;
}

@Override
public boolean isDeleteStashes() {
return true;
}

}
Expand Up @@ -41,7 +41,8 @@ public class MockBlobStoreTest {

@Test
public void smokes() throws Exception {
ArtifactManagerTest.run(j, new JCloudsArtifactManagerFactory(new MockBlobStore()), false, null);
ArtifactManagerTest.artifactArchiveAndDelete(j, new JCloudsArtifactManagerFactory(new MockBlobStore()), false, null);
ArtifactManagerTest.artifactStashAndDelete(j, new JCloudsArtifactManagerFactory(new MockBlobStore()), false, null);
}

}
Expand Up @@ -29,6 +29,7 @@
import io.jenkins.plugins.artifact_manager_jclouds.JCloudsArtifactManagerFactory;
import static org.hamcrest.Matchers.*;
import static org.junit.Assert.*;
import static org.junit.Assume.*;

import java.io.IOException;
import java.io.InputStream;
Expand Down Expand Up @@ -99,16 +100,19 @@ public static void doPrepareImage() throws Exception {
@Rule
public LoggerRule httpLogging = new LoggerRule();

protected ArtifactManagerFactory getArtifactManagerFactory() {
return new JCloudsArtifactManagerFactory(new CustomPrefixBlobStoreProvider(provider, getPrefix()));
protected ArtifactManagerFactory getArtifactManagerFactory(Boolean deleteBlobs, Boolean deleteStashes) {
return new JCloudsArtifactManagerFactory(new CustomPrefixBlobStoreProvider(provider, getPrefix(), deleteBlobs, deleteStashes));
}

private static final class CustomPrefixBlobStoreProvider extends BlobStoreProvider {
private final BlobStoreProvider delegate;
private final String prefix;
CustomPrefixBlobStoreProvider(BlobStoreProvider delegate, String prefix) {
private final Boolean deleteBlobs, deleteStashes;
CustomPrefixBlobStoreProvider(BlobStoreProvider delegate, String prefix, Boolean deleteBlobs, Boolean deleteStashes) {
this.delegate = delegate;
this.prefix = prefix;
this.deleteBlobs = deleteBlobs;
this.deleteStashes = deleteStashes;
}
@Override
public String getPrefix() {
Expand All @@ -119,6 +123,14 @@ public String getContainer() {
return delegate.getContainer();
}
@Override
public boolean isDeleteBlobs() {
return deleteBlobs != null ? deleteBlobs : delegate.isDeleteBlobs();
}
@Override
public boolean isDeleteStashes() {
return deleteStashes != null ? deleteStashes : delegate.isDeleteStashes();
}
@Override
public BlobStoreContext getContext() throws IOException {
return delegate.getContext();
}
Expand All @@ -137,24 +149,43 @@ public BlobStoreProviderDescriptor getDescriptor() {
}

@Test
public void smokes() throws Exception {
if (image != null) {
System.err.println("verifying that while the master can connect to S3, a Dockerized agent cannot");
try (JavaContainer container = image.start(JavaContainer.class).start()) {
DumbSlave agent = new DumbSlave("assumptions", "/home/test/slave", new SSHLauncher(container.ipBound(22), container.port(22), "test", "test", "", ""));
Jenkins.get().addNode(agent);
j.waitOnline(agent);
try {
agent.getChannel().call(new LoadS3Credentials());
fail("did not expect to be able to connect to S3 from a Dockerized agent"); // or AssumptionViolatedException?
} catch (SdkClientException x) {
System.err.println("a Dockerized agent was unable to connect to S3, as expected: " + x);
}
public void agentPermissions() throws Exception {
assumeNotNull(image);
System.err.println("verifying that while the master can connect to S3, a Dockerized agent cannot");
try (JavaContainer container = image.start(JavaContainer.class).start()) {
DumbSlave agent = new DumbSlave("assumptions", "/home/test/slave", new SSHLauncher(container.ipBound(22), container.port(22), "test", "test", "", ""));
Jenkins.get().addNode(agent);
j.waitOnline(agent);
try {
agent.getChannel().call(new LoadS3Credentials());
fail("did not expect to be able to connect to S3 from a Dockerized agent"); // or AssumptionViolatedException?
} catch (SdkClientException x) {
System.err.println("a Dockerized agent was unable to connect to S3, as expected: " + x);
}
}
}

@Test
public void artifactArchive() throws Exception {
// To demo class loading performance: loggerRule.record(SlaveComputer.class, Level.FINEST);
ArtifactManagerTest.run(j, getArtifactManagerFactory(), /* TODO S3BlobStore.list does not seem to handle weird characters */false, image);
ArtifactManagerTest.artifactArchive(j, getArtifactManagerFactory(null, null), /* TODO S3BlobStore.list does not seem to handle weird characters */false, image);
}

@Test
public void artifactArchiveAndDelete() throws Exception {
ArtifactManagerTest.artifactArchiveAndDelete(j, getArtifactManagerFactory(true, null), false, image);
}

@Test
public void artifactStash() throws Exception {
ArtifactManagerTest.artifactStash(j, getArtifactManagerFactory(null, null), false, image);
}

@Test
public void artifactStashAndDelete() throws Exception {
ArtifactManagerTest.artifactStashAndDelete(j, getArtifactManagerFactory(null, true), false, image);
}

private static final class LoadS3Credentials extends MasterToSlaveCallable<Void, RuntimeException> {
@Override
public Void call() {
Expand All @@ -165,7 +196,7 @@ public Void call() {

@Test
public void artifactBrowsingPerformance() throws Exception {
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory());
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null));
FreeStyleProject p = j.createFreeStyleProject();
p.getBuildersList().add(new TestBuilder() {
@Override
Expand Down Expand Up @@ -201,7 +232,7 @@ public boolean perform(AbstractBuild<?, ?> build, Launcher launcher, BuildListen
@Issue({"JENKINS-51390", "JCLOUDS-1200"})
@Test
public void serializationProblem() throws Exception {
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory());
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null));
WorkflowJob p = j.createProject(WorkflowJob.class, "p");
p.setDefinition(new CpsFlowDefinition("node {writeFile file: 'f', text: 'content'; archiveArtifacts 'f'; dir('d') {try {unarchive mapping: ['f': 'f']} catch (x) {sleep 1; echo(/caught $x/)}}}", true));
S3BlobStore.BREAK_CREDS = true;
Expand All @@ -216,7 +247,7 @@ public void serializationProblem() throws Exception {

//@Test
public void archiveSingleLargeFile() throws Exception {
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory());
ArtifactManagerConfiguration.get().getArtifactManagerFactories().add(getArtifactManagerFactory(null, null));
FreeStyleProject p = j.createFreeStyleProject();
p.getBuildersList().add(new TestBuilder() {
@Override
Expand Down

0 comments on commit b09776a

Please sign in to comment.