Skip to content

Commit

Permalink
[JENKINS-51460] Making a proper extension point.
Browse files Browse the repository at this point in the history
  • Loading branch information
@jglick
jglick committed May 21, 2018
1 parent ea62f51 commit cae17e1
Show file tree
Hide file tree
Showing 8 changed files with 150 additions and 71 deletions.
14 changes: 12 additions & 2 deletions src/main/java/io/jenkins/plugins/artifact_manager_s3/BlobStoreProvider.java
View file
Expand Up @@ -37,9 +37,10 @@
import edu.umd.cs.findbugs.annotations.CheckForNull;
import edu.umd.cs.findbugs.annotations.NonNull;
import hudson.ExtensionPoint;
import hudson.model.AbstractDescribableImpl;

@Restricted(Beta.class)
public abstract class BlobStoreProvider implements ExtensionPoint, Serializable {
public abstract class BlobStoreProvider extends AbstractDescribableImpl<BlobStoreProvider> implements ExtensionPoint, Serializable {

private static final long serialVersionUID = -861350249543443493L;

Expand All @@ -48,7 +49,10 @@ public enum HttpMethod {
}

@NonNull
public abstract String id(); // TODO this can go away when it is Describable
public abstract String getPrefix();

@NonNull
public abstract String getContainer();

@NonNull
public abstract BlobStoreContext getContext() throws IOException;
Expand Down Expand Up @@ -79,4 +83,10 @@ public enum HttpMethod {
public URL toExternalURL(@NonNull Blob blob, @NonNull HttpMethod httpMethod) throws IOException {
return null;
}

@Override
public BlobStoreProviderDescriptor getDescriptor() {
return (BlobStoreProviderDescriptor) super.getDescriptor();
}

}
32 changes: 32 additions & 0 deletions src/main/java/io/jenkins/plugins/artifact_manager_s3/BlobStoreProviderDescriptor.java
View file
@@ -0,0 +1,32 @@
/*
* The MIT License
*
* Copyright 2018 CloudBees, Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a copy
* of this software and associated documentation files (the "Software"), to deal
* in the Software without restriction, including without limitation the rights
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
* copies of the Software, and to permit persons to whom the Software is
* furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
* THE SOFTWARE.
*/

package io.jenkins.plugins.artifact_manager_s3;

import hudson.model.Descriptor;

/**
* Descriptor type for {@link BlobStoreProvider}.
*/
public abstract class BlobStoreProviderDescriptor extends Descriptor<BlobStoreProvider> {}
78 changes: 28 additions & 50 deletions src/main/java/io/jenkins/plugins/artifact_manager_s3/JCloudsArtifactManager.java
View file
Expand Up @@ -70,20 +70,18 @@
/**
* Artifact manager that stores files in a JClouds BlobStore using any of JClouds supported backends
*/
class JCloudsArtifactManager extends ArtifactManager implements StashManager.StashAwareArtifactManager {
final class JCloudsArtifactManager extends ArtifactManager implements StashManager.StashAwareArtifactManager {

private static final Logger LOGGER = Logger.getLogger(JCloudsArtifactManager.class.getName());

private static String PROVIDER = System.getProperty("jclouds.provider", "aws-s3");

private static String BLOB_CONTAINER = System.getenv("S3_BUCKET");
private static String PREFIX = System.getenv("S3_DIR");
private final BlobStoreProvider provider;

private transient String key; // e.g. myorg/myrepo/master/123

private transient String prefix;

JCloudsArtifactManager(Run<?, ?> build) {
JCloudsArtifactManager(Run<?, ?> build, BlobStoreProvider provider) {
this.provider = provider;
onLoad(build);
}

Expand All @@ -94,7 +92,7 @@ public void onLoad(Run<?, ?> build) {

// testing only
String getPrefix() {
return prefix == null ? PREFIX : prefix;
return prefix == null ? provider.getPrefix() : prefix;
}

// testing only
Expand All @@ -119,63 +117,61 @@ public void archive(FilePath workspace, Launcher launcher, BuildListener listene
LOGGER.log(Level.FINE, "Archiving from {0}: {1}", new Object[] { workspace, artifacts });
Map<String, URL> artifactUrls = new HashMap<>();
BlobStore blobStore = getContext().getBlobStore();
BlobStoreProvider extension = getExtension(PROVIDER);

// Map artifacts to urls for upload
for (Map.Entry<String, String> entry : artifacts.entrySet()) {
String s3path = "artifacts/" + entry.getKey();
String blobPath = getBlobPath(s3path);
Blob blob = blobStore.blobBuilder(blobPath).build();
blob.getMetadata().setContainer(BLOB_CONTAINER);
artifactUrls.put(entry.getValue(), extension.toExternalURL(blob, HttpMethod.PUT));
blob.getMetadata().setContainer(provider.getContainer());
artifactUrls.put(entry.getValue(), provider.toExternalURL(blob, HttpMethod.PUT));
}

workspace.act(new UploadToBlobStorage(artifactUrls));
listener.getLogger().printf("Uploaded %s artifact(s) to %s%n", artifactUrls.size(), getExtension(PROVIDER).toURI(BLOB_CONTAINER, getBlobPath("artifacts/")));
listener.getLogger().printf("Uploaded %s artifact(s) to %s%n", artifactUrls.size(), provider.toURI(provider.getContainer(), getBlobPath("artifacts/")));
}

@Override
public boolean delete() throws IOException, InterruptedException {
return delete(getContext().getBlobStore(), getBlobPath(""));
return delete(provider, getContext().getBlobStore(), getBlobPath(""));
}

/**
* Delete all blobs starting with prefix
*/
static boolean delete(BlobStore blobStore, String prefix) throws IOException, InterruptedException {
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, BLOB_CONTAINER, ListContainerOptions.Builder.prefix(prefix).recursive());
static boolean delete(BlobStoreProvider provider, BlobStore blobStore, String prefix) throws IOException, InterruptedException {
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(prefix).recursive());
boolean found = false;
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(prefix);
LOGGER.fine("deleting " + path);
blobStore.removeBlob(BLOB_CONTAINER, path);
blobStore.removeBlob(provider.getContainer(), path);
found = true;
}
return found;
}

@Override
public VirtualFile root() {
return new JCloudsVirtualFile(getExtension(PROVIDER), BLOB_CONTAINER, getBlobPath("artifacts"));
return new JCloudsVirtualFile(provider, provider.getContainer(), getBlobPath("artifacts"));
}

@Override
public void stash(String name, FilePath workspace, Launcher launcher, EnvVars env, TaskListener listener, String includes, String excludes, boolean useDefaultExcludes, boolean allowEmpty) throws IOException, InterruptedException {
BlobStoreProvider extension = getExtension(PROVIDER);
BlobStore blobStore = getContext().getBlobStore();

// Map stash to url for upload
String path = getBlobPath("stashes/" + name + ".tgz");
Blob blob = blobStore.blobBuilder(path).build();
blob.getMetadata().setContainer(BLOB_CONTAINER);
URL url = extension.toExternalURL(blob, HttpMethod.PUT);
blob.getMetadata().setContainer(provider.getContainer());
URL url = provider.toExternalURL(blob, HttpMethod.PUT);
int count = workspace.act(new Stash(url, includes, excludes, useDefaultExcludes, WorkspaceList.tempDir(workspace).getRemote()));
if (count == 0 && !allowEmpty) {
throw new AbortException("No files included in stash");
}
listener.getLogger().printf("Stashed %d file(s) to %s%n", count, extension.toURI(BLOB_CONTAINER, path));
listener.getLogger().printf("Stashed %d file(s) to %s%n", count, provider.toURI(provider.getContainer(), path));
}

private static final class Stash extends MasterToSlaveFileCallable<Integer> {
Expand Down Expand Up @@ -219,19 +215,18 @@ public Integer invoke(File f, VirtualChannel channel) throws IOException, Interr

@Override
public void unstash(String name, FilePath workspace, Launcher launcher, EnvVars env, TaskListener listener) throws IOException, InterruptedException {
BlobStoreProvider extension = getExtension(PROVIDER);
BlobStore blobStore = getContext().getBlobStore();

// Map stash to url for download
String blobPath = getBlobPath("stashes/" + name + ".tgz");
Blob blob = blobStore.getBlob(BLOB_CONTAINER, blobPath);
Blob blob = blobStore.getBlob(provider.getContainer(), blobPath);
if (blob == null) {
throw new AbortException(
String.format("No such saved stash ‘%s’ found at %s/%s", name, BLOB_CONTAINER, blobPath));
String.format("No such saved stash ‘%s’ found at %s/%s", name, provider.getContainer(), blobPath));
}
URL url = extension.toExternalURL(blob, HttpMethod.GET);
URL url = provider.toExternalURL(blob, HttpMethod.GET);
workspace.act(new Unstash(url));
listener.getLogger().printf("Unstashed file(s) from %s%n", extension.toURI(BLOB_CONTAINER, blobPath));
listener.getLogger().printf("Unstashed file(s) from %s%n", provider.toURI(provider.getContainer(), blobPath));
}

private static final class Unstash extends MasterToSlaveFileCallable<Void> {
Expand All @@ -255,19 +250,18 @@ public Void invoke(File f, VirtualChannel channel) throws IOException, Interrupt
@Override
public void clearAllStashes(TaskListener listener) throws IOException, InterruptedException {
String stashPrefix = getBlobPath("stashes/");
BlobStoreProvider extension = getExtension(PROVIDER);
BlobStore blobStore = getContext().getBlobStore();
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, BLOB_CONTAINER, ListContainerOptions.Builder.prefix(stashPrefix).recursive());
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(stashPrefix).recursive());
int count = 0;
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(stashPrefix);
LOGGER.fine("deleting " + path);
blobStore.removeBlob(BLOB_CONTAINER, path);
blobStore.removeBlob(provider.getContainer(), path);
count++;
}
listener.getLogger().printf("Deleted %d stash(es) from %s%n", count, extension.toURI(BLOB_CONTAINER, stashPrefix));
listener.getLogger().printf("Deleted %d stash(es) from %s%n", count, provider.toURI(provider.getContainer(), stashPrefix));
}

@Override
Expand All @@ -278,39 +272,23 @@ public void copyAllArtifactsAndStashes(Run<?, ?> to, TaskListener listener) thro
}
JCloudsArtifactManager dest = (JCloudsArtifactManager) am;
String allPrefix = getBlobPath("");
BlobStoreProvider extension = getExtension(PROVIDER);
BlobStore blobStore = getContext().getBlobStore();
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, BLOB_CONTAINER, ListContainerOptions.Builder.prefix(allPrefix).recursive());
Iterator<StorageMetadata> it = new JCloudsVirtualFile.PageSetIterable(blobStore, provider.getContainer(), ListContainerOptions.Builder.prefix(allPrefix).recursive());
int count = 0;
while (it.hasNext()) {
StorageMetadata sm = it.next();
String path = sm.getName();
assert path.startsWith(allPrefix);
String destPath = getBlobPath(dest.key, path.substring(allPrefix.length()));
LOGGER.fine("copying " + path + " to " + destPath);
blobStore.copyBlob(BLOB_CONTAINER, path, BLOB_CONTAINER, destPath, CopyOptions.NONE);
blobStore.copyBlob(provider.getContainer(), path, provider.getContainer(), destPath, CopyOptions.NONE);
count++;
}
listener.getLogger().printf("Copied %d artifact(s)/stash(es) from %s to %s%n", count, extension.toURI(BLOB_CONTAINER, allPrefix), extension.toURI(BLOB_CONTAINER, dest.getBlobPath("")));
}

/**
* Get the extension implementation for the specific JClouds provider or api id
*
* @param providerOrApi
* @throws IllegalStateException
* if extension is not present or run from the agent
* @return the extension implementation
*/
@NonNull
private static BlobStoreProvider getExtension(String providerOrApi) {
return ExtensionList.lookup(BlobStoreProvider.class).stream().filter(e -> providerOrApi.equals(e.id()))
.findFirst()
.orElseThrow(() -> new IllegalStateException("Could not find an extension for " + providerOrApi));
listener.getLogger().printf("Copied %d artifact(s)/stash(es) from %s to %s%n", count, provider.toURI(provider.getContainer(), allPrefix), provider.toURI(provider.getContainer(), dest.getBlobPath("")));
}

private static BlobStoreContext getContext() throws IOException {
return getExtension(PROVIDER).getContext();
private BlobStoreContext getContext() throws IOException {
return provider.getContext();
}

private static class UploadToBlobStorage extends MasterToSlaveFileCallable<Void> {
Expand Down
13 changes: 10 additions & 3 deletions src/main/java/io/jenkins/plugins/artifact_manager_s3/JCloudsArtifactManagerFactory.java
View file
Expand Up @@ -36,21 +36,28 @@
*/
public class JCloudsArtifactManagerFactory extends ArtifactManagerFactory {

private final BlobStoreProvider provider;

@DataBoundConstructor
public JCloudsArtifactManagerFactory() {
public JCloudsArtifactManagerFactory(BlobStoreProvider provider) {
this.provider = provider;
}

public BlobStoreProvider getProvider() {
return provider;
}

@Override
public ArtifactManager managerFor(Run<?, ?> build) {
return new JCloudsArtifactManager(build);
return new JCloudsArtifactManager(build, provider);
}

@Extension
public static final class DescriptorImpl extends ArtifactManagerFactoryDescriptor {

@Override
public String getDisplayName() {
return "S3-based Artifact Storage";
return "Cloud Artifact Storage";
}

}
Expand Down
36 changes: 30 additions & 6 deletions src/main/java/io/jenkins/plugins/artifact_manager_s3/S3BlobStore.java
View file
Expand Up @@ -47,6 +47,7 @@
import org.jclouds.osgi.ProviderRegistry;
import org.kohsuke.accmod.Restricted;
import org.kohsuke.accmod.restrictions.NoExternalUse;
import org.kohsuke.stapler.DataBoundConstructor;

import com.amazonaws.auth.AWSSessionCredentials;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
Expand All @@ -59,25 +60,38 @@
* Extension that customizes JCloudsBlobStore for AWS S3. Credentials are fetched from the environment, env vars, aws
* profiles,...
*/
@Extension
@Restricted(NoExternalUse.class)
public class S3BlobStore extends BlobStoreProvider {

private static final Logger LOGGER = Logger.getLogger(S3BlobStore.class.getName());

private static final long serialVersionUID = -8864075675579867370L;

// For now, these are taken from the environment, rather than being configured.
@SuppressWarnings("FieldMayBeFinal")
private static String BLOB_CONTAINER = System.getenv("S3_BUCKET");
@SuppressWarnings("FieldMayBeFinal")
private static String PREFIX = System.getenv("S3_DIR");

@DataBoundConstructor
public S3BlobStore() {}

@Override
public String getPrefix() {
return PREFIX;
}

@Override
public String id() {
return "aws-s3";
public String getContainer() {
return BLOB_CONTAINER;
}

@Override
public BlobStoreContext getContext() throws IOException {
LOGGER.log(Level.FINEST, "Building context for {0}", id());
LOGGER.log(Level.FINEST, "Building context");
ProviderRegistry.registerProvider(AWSS3ProviderMetadata.builder().build());
try {
return ContextBuilder.newBuilder(id()).credentialsSupplier(getCredentialsSupplier())
return ContextBuilder.newBuilder("aws-s3").credentialsSupplier(getCredentialsSupplier())
.buildView(BlobStoreContext.class);
} catch (NoSuchElementException x) {
throw new IOException(x);
Expand Down Expand Up @@ -139,9 +153,19 @@ public URL toExternalURL(@NonNull Blob blob, @NonNull HttpMethod httpMethod) thr
awsMethod = com.amazonaws.HttpMethod.GET;
break;
default:
throw new IOException("HTTP Method " + httpMethod + " not supported for extension " + id());
throw new IOException("HTTP Method " + httpMethod + " not supported for S3");
}
return builder.build().generatePresignedUrl(container, name, expiration, awsMethod);
}

@Extension
public static final class DescriptorImpl extends BlobStoreProviderDescriptor {

@Override
public String getDisplayName() {
return "Amazon S3";
}

}

}

0 comments on commit cae17e1

Please sign in to comment.