001 /**
002 * Copyright 2010-2013 The Kuali Foundation
003 *
004 * Licensed under the Educational Community License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.opensource.org/licenses/ecl2.php
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016 package org.kuali.maven.wagon;
017
018 import java.io.File;
019 import java.io.FileNotFoundException;
020 import java.io.IOException;
021 import java.io.InputStream;
022 import java.io.OutputStream;
023 import java.net.URI;
024 import java.util.ArrayList;
025 import java.util.Date;
026 import java.util.List;
027
028 import org.apache.commons.io.IOUtils;
029 import org.apache.commons.lang.StringUtils;
030 import org.apache.maven.wagon.ResourceDoesNotExistException;
031 import org.apache.maven.wagon.TransferFailedException;
032 import org.apache.maven.wagon.authentication.AuthenticationException;
033 import org.apache.maven.wagon.authentication.AuthenticationInfo;
034 import org.apache.maven.wagon.proxy.ProxyInfo;
035 import org.apache.maven.wagon.repository.Repository;
036 import org.apache.maven.wagon.repository.RepositoryPermissions;
037 import org.kuali.common.aws.s3.S3Utils;
038 import org.kuali.common.aws.s3.SimpleFormatter;
039 import org.kuali.common.threads.ExecutionStatistics;
040 import org.kuali.common.threads.ThreadHandlerContext;
041 import org.kuali.common.threads.ThreadInvoker;
042 import org.kuali.common.threads.listener.PercentCompleteListener;
043 import org.slf4j.Logger;
044 import org.slf4j.LoggerFactory;
045
046 import com.amazonaws.AmazonClientException;
047 import com.amazonaws.AmazonServiceException;
048 import com.amazonaws.ClientConfiguration;
049 import com.amazonaws.Protocol;
050 import com.amazonaws.auth.AWSCredentials;
051 import com.amazonaws.auth.BasicAWSCredentials;
052 import com.amazonaws.services.s3.AmazonS3Client;
053 import com.amazonaws.services.s3.internal.Mimetypes;
054 import com.amazonaws.services.s3.internal.RepeatableFileInputStream;
055 import com.amazonaws.services.s3.model.CannedAccessControlList;
056 import com.amazonaws.services.s3.model.ListObjectsRequest;
057 import com.amazonaws.services.s3.model.ObjectListing;
058 import com.amazonaws.services.s3.model.ObjectMetadata;
059 import com.amazonaws.services.s3.model.PutObjectRequest;
060 import com.amazonaws.services.s3.model.S3Object;
061 import com.amazonaws.services.s3.model.S3ObjectSummary;
062 import com.amazonaws.services.s3.transfer.TransferManager;
063
064 /**
065 * <p>
066 * An implementation of the Maven Wagon interface that is integrated with the Amazon S3 service.
067 * </p>
068 *
069 * <p>
070 * URLs that reference the S3 service should be in the form of <code>s3://bucket.name</code>. As an example
071 * <code>s3://maven.kuali.org</code> puts files into the <code>maven.kuali.org</code> bucket on the S3 service.
072 * </p>
073 *
074 * <p>
075 * This implementation uses the <code>username</code> and <code>password</code> portions of the server authentication metadata for
076 * credentials.
077 * </p>
078 *
079 * @plexus.component role="org.apache.maven.wagon.Wagon" role-hint="http" instantiation-strategy="per-lookup"
080 *
081 * @author Ben Hale
082 * @author Jeff Caddel
083 */
084 public class S3Wagon extends AbstractWagon implements RequestFactory {
085
086 /**
087 * Set the system property <code>maven.wagon.protocol</code> to <code>http</code> to force the wagon to communicate over
088 * <code>http</code>. Default is <code>https</code>.
089 */
090 public static final String PROTOCOL_KEY = "maven.wagon.protocol";
091 public static final String HTTP = "http";
092 public static final String HTTP_ENDPOINT_VALUE = "http://s3.amazonaws.com";
093 public static final String HTTPS = "https";
094 public static final String MIN_THREADS_KEY = "maven.wagon.threads.min";
095 public static final String MAX_THREADS_KEY = "maven.wagon.threads.max";
096 public static final String DIVISOR_KEY = "maven.wagon.threads.divisor";
097 public static final int DEFAULT_MIN_THREAD_COUNT = 10;
098 public static final int DEFAULT_MAX_THREAD_COUNT = 50;
099 public static final int DEFAULT_DIVISOR = 50;
100 public static final int DEFAULT_READ_TIMEOUT = 60 * 1000;
101 public static final CannedAccessControlList DEFAULT_ACL = CannedAccessControlList.PublicRead;
102
103 ThreadInvoker invoker = new ThreadInvoker();
104 SimpleFormatter formatter = new SimpleFormatter();
105 int minThreads = getMinThreads();
106 int maxThreads = getMaxThreads();
107 int divisor = getDivisor();
108 String protocol = getValue(PROTOCOL_KEY, HTTPS);
109 boolean http = HTTP.equals(protocol);
110 int readTimeout = DEFAULT_READ_TIMEOUT;
111 CannedAccessControlList acl = DEFAULT_ACL;
112 TransferManager transferManager;
113
114 private static final Logger log = LoggerFactory.getLogger(S3Wagon.class);
115
116 private AmazonS3Client client;
117
118 private String bucketName;
119
120 private String basedir;
121
122 private final Mimetypes mimeTypes = Mimetypes.getInstance();
123
124 public S3Wagon() {
125 super(true);
126 S3Listener listener = new S3Listener();
127 super.addSessionListener(listener);
128 super.addTransferListener(listener);
129 }
130
131 protected void validateBucket(AmazonS3Client client, String bucketName) {
132 log.debug("Looking for bucket: " + bucketName);
133 if (client.doesBucketExist(bucketName)) {
134 log.debug("Found bucket '" + bucketName + "' Validating permissions");
135 validatePermissions(client, bucketName);
136 } else {
137 log.info("Creating bucket " + bucketName);
138 // If we create the bucket, we "own" it and by default have the "fullcontrol" permission
139 client.createBucket(bucketName);
140 }
141 }
142
143 /**
144 * Establish that we have enough permissions on this bucket to do what we need to do
145 */
146 protected void validatePermissions(AmazonS3Client client, String bucketName) {
147 // This establishes our ability to list objects in this bucket
148 ListObjectsRequest zeroObjectsRequest = new ListObjectsRequest(bucketName, null, null, null, 0);
149 client.listObjects(zeroObjectsRequest);
150
151 /**
152 * The current AWS Java SDK does not appear to have a simple method for discovering what set of permissions the currently
153 * authenticated user has on a bucket. The AWS dev's suggest that you attempt to perform an operation that would fail if you don't
154 * have the permission in question. You would then use the success/failure of that attempt to establish what your permissions are.
155 * This is definitely not ideal and they are working on it, but it is not ready yet.
156 */
157
158 // Do something simple and quick to verify that we have write permissions on this bucket
159 // One way to do this would be to create an object in this bucket, and then immediately delete it
160 // That seems messy, inconvenient, and lame.
161
162 }
163
164 protected CannedAccessControlList getAclFromRepository(Repository repository) {
165 RepositoryPermissions permissions = repository.getPermissions();
166 if (permissions == null) {
167 return null;
168 }
169 String filePermissions = permissions.getFileMode();
170 if (StringUtils.isBlank(filePermissions)) {
171 return null;
172 }
173 return CannedAccessControlList.valueOf(filePermissions.trim());
174 }
175
176 protected ClientConfiguration getClientConfiguration() {
177 ClientConfiguration configuration = new ClientConfiguration();
178 if (http) {
179 log.info("http selected");
180 configuration.setProtocol(Protocol.HTTP);
181 }
182 return configuration;
183 }
184
185 protected AmazonS3Client getAmazonS3Client(AWSCredentials credentials) {
186 ClientConfiguration configuration = getClientConfiguration();
187 return new AmazonS3Client(credentials, configuration);
188 }
189
190 @Override
191 protected void connectToRepository(Repository source, AuthenticationInfo auth, ProxyInfo proxy) throws AuthenticationException {
192
193 AWSCredentials credentials = getCredentials(auth);
194 this.client = getAmazonS3Client(credentials);
195 this.transferManager = new TransferManager(credentials);
196 this.bucketName = source.getHost();
197 validateBucket(client, bucketName);
198 this.basedir = getBaseDir(source);
199
200 // If they've specified <filePermissions> in settings.xml, that always wins
201 CannedAccessControlList repoAcl = getAclFromRepository(source);
202 if (repoAcl != null) {
203 log.info("File permissions: " + repoAcl.name());
204 acl = repoAcl;
205 }
206 }
207
208 @Override
209 protected boolean doesRemoteResourceExist(final String resourceName) {
210 try {
211 client.getObjectMetadata(bucketName, basedir + resourceName);
212 } catch (AmazonClientException e) {
213 return false;
214 }
215 return true;
216 }
217
218 @Override
219 protected void disconnectFromRepository() {
220 // Nothing to do for S3
221 }
222
223 /**
224 * Pull an object out of an S3 bucket and write it to a file
225 */
226 @Override
227 protected void getResource(final String resourceName, final File destination, final TransferProgress progress) throws ResourceDoesNotExistException, IOException {
228 // Obtain the object from S3
229 S3Object object = null;
230 try {
231 String key = basedir + resourceName;
232 object = client.getObject(bucketName, key);
233 } catch (Exception e) {
234 throw new ResourceDoesNotExistException("Resource " + resourceName + " does not exist in the repository", e);
235 }
236
237 //
238 InputStream in = null;
239 OutputStream out = null;
240 try {
241 in = object.getObjectContent();
242 out = new TransferProgressFileOutputStream(destination, progress);
243 byte[] buffer = new byte[1024];
244 int length;
245 while ((length = in.read(buffer)) != -1) {
246 out.write(buffer, 0, length);
247 }
248 } finally {
249 IOUtils.closeQuietly(in);
250 IOUtils.closeQuietly(out);
251 }
252 }
253
254 /**
255 * Is the S3 object newer than the timestamp passed in?
256 */
257 @Override
258 protected boolean isRemoteResourceNewer(final String resourceName, final long timestamp) {
259 ObjectMetadata metadata = client.getObjectMetadata(bucketName, basedir + resourceName);
260 return metadata.getLastModified().compareTo(new Date(timestamp)) < 0;
261 }
262
263 /**
264 * List all of the objects in a given directory
265 */
266 @Override
267 protected List<String> listDirectory(String directory) throws Exception {
268 // info("directory=" + directory);
269 if (StringUtils.isBlank(directory)) {
270 directory = "";
271 }
272 String delimiter = "/";
273 String prefix = basedir + directory;
274 if (!prefix.endsWith(delimiter)) {
275 prefix += delimiter;
276 }
277 // info("prefix=" + prefix);
278 ListObjectsRequest request = new ListObjectsRequest();
279 request.setBucketName(bucketName);
280 request.setPrefix(prefix);
281 request.setDelimiter(delimiter);
282 ObjectListing objectListing = client.listObjects(request);
283 // info("truncated=" + objectListing.isTruncated());
284 // info("prefix=" + prefix);
285 // info("basedir=" + basedir);
286 List<String> fileNames = new ArrayList<String>();
287 for (S3ObjectSummary summary : objectListing.getObjectSummaries()) {
288 // info("summary.getKey()=" + summary.getKey());
289 String key = summary.getKey();
290 String relativeKey = key.startsWith(basedir) ? key.substring(basedir.length()) : key;
291 boolean add = !StringUtils.isBlank(relativeKey) && !relativeKey.equals(directory);
292 if (add) {
293 // info("Adding key - " + relativeKey);
294 fileNames.add(relativeKey);
295 }
296 }
297 for (String commonPrefix : objectListing.getCommonPrefixes()) {
298 String value = commonPrefix.startsWith(basedir) ? commonPrefix.substring(basedir.length()) : commonPrefix;
299 // info("commonPrefix=" + commonPrefix);
300 // info("relativeValue=" + relativeValue);
301 // info("Adding common prefix - " + value);
302 fileNames.add(value);
303 }
304 // StringBuilder sb = new StringBuilder();
305 // sb.append("\n");
306 // for (String fileName : fileNames) {
307 // sb.append(fileName + "\n");
308 // }
309 // info(sb.toString());
310 return fileNames;
311 }
312
313 protected void info(String msg) {
314 System.out.println("[INFO] " + msg);
315 }
316
317 /**
318 * Normalize the key to our S3 object<br>
319 * 1. Convert "./css/style.css" into "/css/style.css"<br>
320 * 2. Convert "/foo/bar/../../css/style.css" into "/css/style.css"
321 *
322 * @see java.net.URI.normalize()
323 */
324 protected String getNormalizedKey(final File source, final String destination) {
325 // Generate our bucket key for this file
326 String key = basedir + destination;
327 try {
328 String prefix = "http://s3.amazonaws.com/" + bucketName + "/";
329 String urlString = prefix + key;
330 URI rawURI = new URI(urlString);
331 URI normalizedURI = rawURI.normalize();
332 String normalized = normalizedURI.toString();
333 int pos = normalized.indexOf(prefix) + prefix.length();
334 String normalizedKey = normalized.substring(pos);
335 return normalizedKey;
336 } catch (Exception e) {
337 throw new RuntimeException(e);
338 }
339 }
340
341 protected ObjectMetadata getObjectMetadata(final File source, final String destination) {
342 // Set the mime type according to the extension of the destination file
343 String contentType = mimeTypes.getMimetype(destination);
344 long contentLength = source.length();
345
346 ObjectMetadata omd = new ObjectMetadata();
347 omd.setContentLength(contentLength);
348 omd.setContentType(contentType);
349 return omd;
350 }
351
352 /**
353 * Create a PutObjectRequest based on the PutContext
354 */
355 public PutObjectRequest getPutObjectRequest(PutFileContext context) {
356 File source = context.getSource();
357 String destination = context.getDestination();
358 TransferProgress progress = context.getProgress();
359 return getPutObjectRequest(source, destination, progress);
360 }
361
362 protected InputStream getInputStream(File source, TransferProgress progress) throws FileNotFoundException {
363 if (progress == null) {
364 return new RepeatableFileInputStream(source);
365 } else {
366 return new TransferProgressFileInputStream(source, progress);
367 }
368 }
369
370 /**
371 * Create a PutObjectRequest based on the source file and destination passed in
372 */
373 protected PutObjectRequest getPutObjectRequest(File source, String destination, TransferProgress progress) {
374 try {
375 String key = getNormalizedKey(source, destination);
376 InputStream input = getInputStream(source, progress);
377 ObjectMetadata metadata = getObjectMetadata(source, destination);
378 PutObjectRequest request = new PutObjectRequest(bucketName, key, input, metadata);
379 request.setCannedAcl(acl);
380 return request;
381 } catch (FileNotFoundException e) {
382 throw new AmazonServiceException("File not found", e);
383 }
384 }
385
386 /**
387 * On S3 there are no true "directories". An S3 bucket is essentially a Hashtable of files stored by key. The integration between a
388 * traditional file system and an S3 bucket is to use the path of the file on the local file system as the key to the file in the
389 * bucket. The S3 bucket does not contain a separate key for the directory itself.
390 */
391 public final void putDirectory(File sourceDir, String destinationDir) throws TransferFailedException {
392
393 // Examine the contents of the directory
394 List<PutFileContext> contexts = getPutFileContexts(sourceDir, destinationDir);
395 for (PutFileContext context : contexts) {
396 // Progress is tracked by the thread handler when uploading files this way
397 context.setProgress(null);
398 }
399
400 // Sum the total bytes in the directory
401 long bytes = sum(contexts);
402
403 // Show what we are up to
404 log.info(getUploadStartMsg(contexts.size(), bytes));
405
406 // Store some context for the thread handler
407 ThreadHandlerContext<PutFileContext> thc = new ThreadHandlerContext<PutFileContext>();
408 thc.setList(contexts);
409 thc.setHandler(new FileHandler());
410 thc.setMax(maxThreads);
411 thc.setMin(minThreads);
412 thc.setDivisor(divisor);
413 thc.setListener(new PercentCompleteListener<PutFileContext>());
414
415 // Invoke the threads
416 ExecutionStatistics stats = invoker.invokeThreads(thc);
417
418 // Show some stats
419 long millis = stats.getExecutionTime();
420 long count = stats.getIterationCount();
421 log.info(getUploadCompleteMsg(millis, bytes, count));
422 }
423
424 protected String getUploadCompleteMsg(long millis, long bytes, long count) {
425 String rate = formatter.getRate(millis, bytes);
426 String time = formatter.getTime(millis);
427 StringBuilder sb = new StringBuilder();
428 sb.append("Files: " + count);
429 sb.append(" Time: " + time);
430 sb.append(" Rate: " + rate);
431 return sb.toString();
432 }
433
434 protected String getUploadStartMsg(int fileCount, long bytes) {
435 StringBuilder sb = new StringBuilder();
436 sb.append("Files: " + fileCount);
437 sb.append(" Bytes: " + formatter.getSize(bytes));
438 return sb.toString();
439 }
440
441 protected int getRequestsPerThread(int threads, int requests) {
442 int requestsPerThread = requests / threads;
443 while (requestsPerThread * threads < requests) {
444 requestsPerThread++;
445 }
446 return requestsPerThread;
447 }
448
449 protected long sum(List<PutFileContext> contexts) {
450 long sum = 0;
451 for (PutFileContext context : contexts) {
452 File file = context.getSource();
453 long length = file.length();
454 sum += length;
455 }
456 return sum;
457 }
458
459 /**
460 * Store a resource into S3
461 */
462 @Override
463 protected void putResource(final File source, final String destination, final TransferProgress progress) throws IOException {
464
465 // Create a new PutObjectRequest
466 PutObjectRequest request = getPutObjectRequest(source, destination, progress);
467
468 // Upload the file to S3, using multi-part upload for large files
469 S3Utils.getInstance().upload(source, request, client, transferManager);
470 }
471
472 protected String getDestinationPath(final String destination) {
473 return destination.substring(0, destination.lastIndexOf('/'));
474 }
475
476 /**
477 * Convert "/" -> ""<br>
478 * Convert "/snapshot/" -> "snapshot/"<br>
479 * Convert "/snapshot" -> "snapshot/"<br>
480 */
481 protected String getBaseDir(final Repository source) {
482 StringBuilder sb = new StringBuilder(source.getBasedir());
483 sb.deleteCharAt(0);
484 if (sb.length() == 0) {
485 return "";
486 }
487 if (sb.charAt(sb.length() - 1) != '/') {
488 sb.append('/');
489 }
490 return sb.toString();
491 }
492
493 protected String getAuthenticationErrorMessage() {
494 StringBuffer sb = new StringBuffer();
495 sb.append("The S3 wagon needs AWS Access Key set as the username and AWS Secret Key set as the password. eg:\n");
496 sb.append("<server>\n");
497 sb.append(" <id>my.server</id>\n");
498 sb.append(" <username>[AWS Access Key ID]</username>\n");
499 sb.append(" <password>[AWS Secret Access Key]</password>\n");
500 sb.append("</server>\n");
501 return sb.toString();
502 }
503
504 /**
505 * Create AWSCredentionals from the information in settings.xml
506 */
507 protected AWSCredentials getCredentials(final AuthenticationInfo authenticationInfo) throws AuthenticationException {
508 if (authenticationInfo == null) {
509 throw new AuthenticationException(getAuthenticationErrorMessage());
510 }
511 String accessKey = authenticationInfo.getUserName();
512 String secretKey = authenticationInfo.getPassword();
513 if (accessKey == null || secretKey == null) {
514 throw new AuthenticationException(getAuthenticationErrorMessage());
515 }
516 return new BasicAWSCredentials(accessKey, secretKey);
517 }
518
519 /*
520 * (non-Javadoc)
521 *
522 * @see org.kuali.maven.wagon.AbstractWagon#getPutFileContext(java.io.File, java.lang.String)
523 */
524 @Override
525 protected PutFileContext getPutFileContext(File source, String destination) {
526 PutFileContext context = super.getPutFileContext(source, destination);
527 context.setFactory(this);
528 context.setTransferManager(this.transferManager);
529 context.setClient(this.client);
530 return context;
531 }
532
533 protected int getMinThreads() {
534 return getValue(MIN_THREADS_KEY, DEFAULT_MIN_THREAD_COUNT);
535 }
536
537 protected int getMaxThreads() {
538 return getValue(MAX_THREADS_KEY, DEFAULT_MAX_THREAD_COUNT);
539 }
540
541 protected int getDivisor() {
542 return getValue(DIVISOR_KEY, DEFAULT_DIVISOR);
543 }
544
545 protected int getValue(String key, int defaultValue) {
546 String value = System.getProperty(key);
547 if (StringUtils.isEmpty(value)) {
548 return defaultValue;
549 } else {
550 return new Integer(value);
551 }
552 }
553
554 protected String getValue(String key, String defaultValue) {
555 String value = System.getProperty(key);
556 if (StringUtils.isEmpty(value)) {
557 return defaultValue;
558 } else {
559 return value;
560 }
561 }
562
563 public int getReadTimeout() {
564 return readTimeout;
565 }
566
567 public void setReadTimeout(int readTimeout) {
568 this.readTimeout = readTimeout;
569 }
570
571 }