svn commit: r1580783 - in /oodt/trunk/filemgr: ./ src/main/bin/ src/main/java/org/apache/oodt/cas/filemgr/datatransfer/ src/main/resources/ src/test/org/apache/oodt/cas/filemgr/datatransfer/

[bf...@apache.org]

Author: bfoster
Date: Mon Mar 24 08:26:48 2014
New Revision: 1580783

URL: http://svn.apache.org/r1580783
Log:
- Updates to Amazon S3 DataTransferer

Added:
    oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java   (with props)
Modified:
    oodt/trunk/filemgr/pom.xml
    oodt/trunk/filemgr/src/main/bin/filemgr
    oodt/trunk/filemgr/src/main/bin/filemgr-client
    oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
    oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
    oodt/trunk/filemgr/src/main/resources/filemgr.properties

Modified: oodt/trunk/filemgr/pom.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/pom.xml?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/pom.xml (original)
+++ oodt/trunk/filemgr/pom.xml Mon Mar 24 08:26:48 2014
@@ -282,7 +282,7 @@
 	<dependency>
       <groupId>com.amazonaws</groupId>
       <artifactId>aws-java-sdk</artifactId>
-      <version>1.5.7</version>
+      <version>1.7.4</version>
 	</dependency>
     <dependency>
       <groupId>junit</groupId>

Modified: oodt/trunk/filemgr/src/main/bin/filemgr
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/bin/filemgr?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/bin/filemgr (original)
+++ oodt/trunk/filemgr/src/main/bin/filemgr Mon Mar 24 08:26:48 2014
@@ -48,12 +48,16 @@ export PATH
 ## just to be on the safe side
 mkdir -p ${RUN_HOME}
 
+for file in `find ../lib/*.jar`; do
+     LIB_DEPS="${file}:${LIB_DEPS}"
+done
+
 # See how we were called.
 case "$1" in
   start)
         echo -n "Starting cas file manager: "
         $JAVA_HOME/bin/java \
-        	-Djava.ext.dirs=${CAS_FILEMGR_HOME}/lib \
+        	-cp ${LIB_DEPS} \
         	-Djava.util.logging.config.file=${CAS_FILEMGR_HOME}/etc/logging.properties \
     	    -Dorg.apache.oodt.cas.filemgr.properties=${CAS_FILEMGR_PROPS} \
         	org.apache.oodt.cas.filemgr.system.XmlRpcFileManager \

Modified: oodt/trunk/filemgr/src/main/bin/filemgr-client
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/bin/filemgr-client?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/bin/filemgr-client (original)
+++ oodt/trunk/filemgr/src/main/bin/filemgr-client Mon Mar 24 08:26:48 2014
@@ -24,10 +24,14 @@ fi
 
 export JAVA_HOME
 
+for file in `find ../lib/*.jar`; do
+     LIB_DEPS="${file}:${LIB_DEPS}"
+done
+
 $JAVA_HOME/bin/java \
-	-Djava.ext.dirs=../lib \
+	-cp ${LIB_DEPS} \
 	-Dorg.apache.oodt.cas.filemgr.properties=../etc/filemgr.properties \
 	-Djava.util.logging.config.file=../etc/logging.properties \
-  -Dorg.apache.oodt.cas.cli.action.spring.config=../policy/cmd-line-actions.xml \
-  -Dorg.apache.oodt.cas.cli.option.spring.config=../policy/cmd-line-options.xml \
+	-Dorg.apache.oodt.cas.cli.action.spring.config=../policy/cmd-line-actions.xml \
+	-Dorg.apache.oodt.cas.cli.option.spring.config=../policy/cmd-line-options.xml \
 	org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient $*

Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java Mon Mar 24 08:26:48 2014
@@ -21,6 +21,8 @@ import static com.google.common.base.Pre
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.net.URL;
 
 import org.apache.oodt.cas.filemgr.structs.Product;
@@ -56,13 +58,14 @@ public class S3DataTransferer implements
 	@Override
 	public void transferProduct(Product product) throws DataTransferException, IOException {
 		for (Reference ref : product.getProductReferences()) {
+      String origRef = stripProtocol(ref.getOrigReference(), false);
+		  String dataStoreRef = stripProtocol(ref.getDataStoreReference(), true);
 			try {
-				s3Client.putObject(new PutObjectRequest(bucketName, ref.getDataStoreReference(), new File(ref
-				    .getOrigReference())));
+				s3Client.putObject(new PutObjectRequest(bucketName, dataStoreRef, new File(origRef)));
 			} catch (AmazonClientException e) {
 				throw new DataTransferException(String.format(
-				    "Failed to upload product reference %s to S3 at %s", ref.getOrigReference(),
-				    ref.getDataStoreReference()), e);
+				    "Failed to upload product reference %s to S3 at %s", origRef,
+				    dataStoreRef), e);
 			}
 		}
 	}
@@ -71,7 +74,7 @@ public class S3DataTransferer implements
 	public void retrieveProduct(Product product, File directory) throws DataTransferException,
 	    IOException {
 		for (Reference ref : product.getProductReferences()) {
-			GetObjectRequest request = new GetObjectRequest(bucketName, ref.getDataStoreReference());
+			GetObjectRequest request = new GetObjectRequest(bucketName, stripProtocol(ref.getDataStoreReference(), true));
 			S3Object file = s3Client.getObject(request);
 			stageFile(file, ref, directory);
 		}
@@ -83,7 +86,7 @@ public class S3DataTransferer implements
 		try {
 			inStream = file.getObjectContent();
 			outStream = new FileOutputStream(new File(directory, new File(
-			    ref.getDataStoreReference()).getName()));
+			    stripProtocol(ref.getDataStoreReference(), false)).getName()));
 			IOUtils.copy(inStream, outStream);
 		} catch (IOException e) {
 			throw e;
@@ -92,4 +95,16 @@ public class S3DataTransferer implements
 			try { outStream.close(); } catch (Exception e) {}
 		}
 	}
+
+	private String stripProtocol(String ref, boolean stripLeadingSeparator) throws IOException {
+	  try {
+      String path = new URI(ref).getPath();
+      if (stripLeadingSeparator) {
+        path = path.replaceAll("^/", "");
+      }
+      return path;
+    } catch (URISyntaxException e) {
+      throw new IOException(e);
+    }
+	}
 }

Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java Mon Mar 24 08:26:48 2014
@@ -19,7 +19,6 @@ package org.apache.oodt.cas.filemgr.data
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.regions.Region;
 import com.amazonaws.regions.Regions;
-import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
 
 /**
@@ -39,15 +38,15 @@ public class S3DataTransfererFactory imp
 			"org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key";
 
 	@Override
-  public DataTransfer createDataTransfer() {
+  public S3DataTransferer createDataTransfer() {
 		String bucketName = System.getProperty(BUCKET_NAME_PROPERTY);
-		String region = System.getProperty(REGION_PROPERTY);
+		String region = System.getProperty(REGION_PROPERTY);		
 		String accessKey = System.getProperty(ACCESS_KEY_PROPERTY);
-		String secretKey = System.getProperty(SECRET_KEY_PROPERTY);
+    String secretKey = System.getProperty(SECRET_KEY_PROPERTY);
 
-		AmazonS3 s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
+		AmazonS3Client s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
     s3.setRegion(Region.getRegion(Regions.valueOf(region)));
 
-	  return new S3DataTransferer(s3, bucketName);
+    return new S3DataTransferer(s3, bucketName);
   }
 }

Modified: oodt/trunk/filemgr/src/main/resources/filemgr.properties
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/resources/filemgr.properties?rev=1580783&r1=1580782&r2=1580783&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/resources/filemgr.properties (original)
+++ oodt/trunk/filemgr/src/main/resources/filemgr.properties Mon Mar 24 08:26:48 2014
@@ -118,7 +118,7 @@ org.apache.oodt.cas.filemgr.datatransfer
 # Region can be any of the values found here:
 #   http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/regions/Regions.html
 org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name=some_bucket_name
-org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1 
+org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1
 org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key
 org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key
 

Added: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java?rev=1580783&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java (added)
+++ oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java Mon Mar 24 08:26:48 2014
@@ -0,0 +1,41 @@
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/**
+ * Test class for {@link S3DataTransfererFactory}.
+ * 
+ * @author bfoster@apache.org (Brian Foster)
+ */
+@RunWith(JUnit4.class)
+public class TestS3DataTransfererFactory {
+
+  private static final String BUCKET_NAME_PROPERTY =
+      "org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name";
+  private static final String REGION_PROPERTY =
+      "org.apache.oodt.cas.filemgr.datatransfer.s3.region";
+  private static final String ACCESS_KEY_PROPERTY =
+      "org.apache.oodt.cas.filemgr.datatransfer.s3.access.key";
+  private static final String SECRET_KEY_PROPERTY =
+      "org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key";
+
+  @Test
+  public void testCreateDataTransferer() {
+    System.setProperty(BUCKET_NAME_PROPERTY, "test-bucket");
+    System.setProperty(REGION_PROPERTY, "US_WEST_1");
+    System.setProperty(ACCESS_KEY_PROPERTY, "23123123123");
+    System.setProperty(SECRET_KEY_PROPERTY, "00101010101");
+
+    S3DataTransfererFactory factory = new S3DataTransfererFactory();
+    S3DataTransferer transferer = factory.createDataTransfer();
+
+    assertThat(transferer, is(not(nullValue())));
+  }
+}

Propchange: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransfererFactory.java
------------------------------------------------------------------------------
    svn:mime-type = text/plain