You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oodt.apache.org by bf...@apache.org on 2014/03/18 07:45:00 UTC
svn commit: r1578758 - in /oodt/trunk/filemgr: ./
src/main/java/org/apache/oodt/cas/filemgr/datatransfer/
src/main/java/org/apache/oodt/cas/filemgr/structs/exceptions/
src/main/resources/ src/test/org/apache/oodt/cas/filemgr/datatransfer/
Author: bfoster
Date: Tue Mar 18 06:44:59 2014
New Revision: 1578758
URL: http://svn.apache.org/r1578758
Log:
- Adds DataTranferer which puts files into S3
Added:
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java (with props)
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java (with props)
oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java (with props)
Modified:
oodt/trunk/filemgr/pom.xml
oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/exceptions/CatalogException.java
oodt/trunk/filemgr/src/main/resources/filemgr.properties
Modified: oodt/trunk/filemgr/pom.xml
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/pom.xml?rev=1578758&r1=1578757&r2=1578758&view=diff
==============================================================================
--- oodt/trunk/filemgr/pom.xml (original)
+++ oodt/trunk/filemgr/pom.xml Tue Mar 18 06:44:59 2014
@@ -48,7 +48,7 @@
<property>
<name>java.util.logging.config.file</name>
<value>${basedir}/src/testdata/test.logging.properties</value>
- </property>
+ </property>
</systemProperties>
<forkedProcessTimeoutInSeconds>0</forkedProcessTimeoutInSeconds>
<redirectTestOutputToFile>true</redirectTestOutputToFile>
@@ -112,6 +112,18 @@
</build>
</profile>
</profiles>
+ <repositories>
+ <repository>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <id>sonatype-nexus</id>
+ <url>https://oss.sonatype.org/content/groups/public</url>
+ </repository>
+ </repositories>
<dependencies>
<dependency>
<groupId>org.apache.oodt</groupId>
@@ -267,6 +279,28 @@
<version>5.1.26</version>
<scope>test</scope>
</dependency>
-
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.5.7</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.9.5</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java?rev=1578758&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java Tue Mar 18 06:44:59 2014
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URL;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
+import org.apache.tika.io.IOUtils;
+
+import com.amazonaws.AmazonClientException;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+
+/**
+ * {@link DataTransfer} which put/gets files in/from Amazon S3 storage.
+ *
+ * @author bfoster@apache.org (Brian Foster)
+ */
+public class S3DataTransferer implements DataTransfer {
+
+ private final AmazonS3 s3Client;
+ private final String bucketName;
+
+ public S3DataTransferer(AmazonS3 s3Client, String bucketName) {
+ this.s3Client = checkNotNull(s3Client);
+ this.bucketName = checkNotNull(bucketName);
+ }
+
+ @Override
+ public void setFileManagerUrl(URL url) {}
+
+ @Override
+ public void transferProduct(Product product) throws DataTransferException, IOException {
+ for (Reference ref : product.getProductReferences()) {
+ try {
+ s3Client.putObject(new PutObjectRequest(bucketName, ref.getDataStoreReference(), new File(ref
+ .getOrigReference())));
+ } catch (AmazonClientException e) {
+ throw new DataTransferException(String.format(
+ "Failed to upload product reference %s to S3 at %s", ref.getOrigReference(),
+ ref.getDataStoreReference()), e);
+ }
+ }
+ }
+
+ @Override
+ public void retrieveProduct(Product product, File directory) throws DataTransferException,
+ IOException {
+ for (Reference ref : product.getProductReferences()) {
+ GetObjectRequest request = new GetObjectRequest(bucketName, ref.getDataStoreReference());
+ S3Object file = s3Client.getObject(request);
+ stageFile(file, ref, directory);
+ }
+ }
+
+ private void stageFile(S3Object file, Reference ref, File directory) throws IOException {
+ S3ObjectInputStream inStream = null;
+ FileOutputStream outStream = null;
+ try {
+ inStream = file.getObjectContent();
+ outStream = new FileOutputStream(new File(directory, new File(
+ ref.getDataStoreReference()).getName()));
+ IOUtils.copy(inStream, outStream);
+ } catch (IOException e) {
+ throw e;
+ } finally {
+ try { inStream.close(); } catch (Exception e) {}
+ try { outStream.close(); } catch (Exception e) {}
+ }
+ }
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransferer.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Added: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java?rev=1578758&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java (added)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java Tue Mar 18 06:44:59 2014
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.regions.Region;
+import com.amazonaws.regions.Regions;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+
+/**
+ * {@link DataTransferFactory} which creates {@link S3DataTransferer}s.
+ *
+ * @author bfoster@apache.org (Brian Foster)
+ */
+public class S3DataTransfererFactory implements DataTransferFactory {
+
+ private static final String BUCKET_NAME_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name";
+ private static final String REGION_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.region";
+ private static final String ACCESS_KEY_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.access.key";
+ private static final String SECRET_KEY_PROPERTY =
+ "org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key";
+
+ @Override
+ public DataTransfer createDataTransfer() {
+ String bucketName = System.getProperty(BUCKET_NAME_PROPERTY);
+ String region = System.getProperty(REGION_PROPERTY);
+ String accessKey = System.getProperty(ACCESS_KEY_PROPERTY);
+ String secretKey = System.getProperty(SECRET_KEY_PROPERTY);
+
+ AmazonS3 s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey));
+ s3.setRegion(Region.getRegion(Regions.valueOf(region)));
+
+ return new S3DataTransferer(s3, bucketName);
+ }
+}
Propchange: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/datatransfer/S3DataTransfererFactory.java
------------------------------------------------------------------------------
svn:mime-type = text/plain
Modified: oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/exceptions/CatalogException.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/exceptions/CatalogException.java?rev=1578758&r1=1578757&r2=1578758&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/exceptions/CatalogException.java (original)
+++ oodt/trunk/filemgr/src/main/java/org/apache/oodt/cas/filemgr/structs/exceptions/CatalogException.java Tue Mar 18 06:44:59 2014
@@ -29,19 +29,15 @@ public class CatalogException extends Ex
private static final long serialVersionUID = 3690753990686029110L;
- /**
- * <p>Default Constructor</p>
- */
public CatalogException() {
super();
}
- /**
- * @param message The message for this exception.
- */
public CatalogException(String message) {
super(message);
}
-
+ public CatalogException(String message, Throwable t) {
+ super(message, t);
+ }
}
Modified: oodt/trunk/filemgr/src/main/resources/filemgr.properties
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/main/resources/filemgr.properties?rev=1578758&r1=1578757&r2=1578758&view=diff
==============================================================================
--- oodt/trunk/filemgr/src/main/resources/filemgr.properties (original)
+++ oodt/trunk/filemgr/src/main/resources/filemgr.properties Tue Mar 18 06:44:59 2014
@@ -114,6 +114,14 @@ org.apache.oodt.cas.filemgr.validation.s
# remote data transfer configuration
org.apache.oodt.cas.filemgr.datatransfer.remote.chunkSize=1024
+# Amazon S3 data transfer configuration.
+# Region can be any of the values found here:
+# http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/regions/Regions.html
+org.apache.oodt.cas.filemgr.datatransfer.s3.bucket.name=some_bucket_name
+org.apache.oodt.cas.filemgr.datatransfer.s3.region=EU_WEST_1
+org.apache.oodt.cas.filemgr.datatransfer.s3.access.key=s3_access_key
+org.apache.oodt.cas.filemgr.datatransfer.s3.secret.key=s3_secret_key
+
# location of Mime-Type repository
org.apache.oodt.cas.filemgr.mime.type.repository=/path/to/mime-types.xml
Added: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java
URL: http://svn.apache.org/viewvc/oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java?rev=1578758&view=auto
==============================================================================
--- oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java (added)
+++ oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java Tue Mar 18 06:44:59 2014
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oodt.cas.filemgr.datatransfer;
+
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.oodt.cas.filemgr.structs.Product;
+import org.apache.oodt.cas.filemgr.structs.Reference;
+import org.apache.oodt.cas.filemgr.structs.exceptions.DataTransferException;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.PutObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.google.common.collect.Lists;
+
+/**
+ * Test class for {@link S3DataTransferer}.
+ *
+ * @author bfoster@apache.org (Brian Foster)
+ */
+@RunWith(JUnit4.class)
+public class TestS3DataTransferer {
+
+ private final static String S3_BUCKET_NAME = "TestBucket";
+ private final static String ORGINAL_REF = "/path/to/file";
+ private final static String DATA_STORE_REF = "/path/in/s3/storage/file";
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+
+ @Mock private AmazonS3 s3Client;
+ @Mock private Product product;
+ @Mock private Reference reference;
+ @Mock private S3Object s3Object;
+ @Mock private S3ObjectInputStream s3InputStream;
+
+ private S3DataTransferer dataTransferer;
+ private File stagingDir;
+
+ @Before
+ public void setUp() throws IOException {
+ MockitoAnnotations.initMocks(this);
+
+ stagingDir = tempFolder.getRoot();
+ dataTransferer = new S3DataTransferer(s3Client, S3_BUCKET_NAME);
+
+ when(reference.getOrigReference()).thenReturn(ORGINAL_REF);
+ when(reference.getDataStoreReference()).thenReturn(DATA_STORE_REF);
+ when(product.getProductReferences()).thenReturn(Lists.newArrayList(reference));
+ when(s3Client.getObject(Mockito.<GetObjectRequest>any())).thenReturn(s3Object);
+ when(s3Object.getObjectContent()).thenReturn(s3InputStream);
+ when(s3InputStream.read(Mockito.<byte[]>any())).thenReturn(-1);
+ }
+
+ @Test
+ public void testTransferProduct() throws DataTransferException, IOException {
+ dataTransferer.transferProduct(product);
+
+ ArgumentCaptor<PutObjectRequest> argument = ArgumentCaptor.forClass(PutObjectRequest.class);
+ verify(s3Client).putObject(argument.capture());
+
+ PutObjectRequest request = argument.getValue();
+ assertThat(request.getBucketName(), is(S3_BUCKET_NAME));
+ assertThat(request.getKey(), is(DATA_STORE_REF));
+ assertThat(request.getFile().getAbsolutePath(), is(ORGINAL_REF));
+ }
+
+ @Test
+ public void testRetrieveProduct() throws DataTransferException, IOException {
+ dataTransferer.retrieveProduct(product, stagingDir);
+
+ ArgumentCaptor<GetObjectRequest> argument = ArgumentCaptor.forClass(GetObjectRequest.class);
+ verify(s3Client).getObject(argument.capture());
+
+ GetObjectRequest request = argument.getValue();
+ assertThat(request.getBucketName(), is(S3_BUCKET_NAME));
+ assertThat(request.getKey(), is(DATA_STORE_REF));
+ }
+}
Propchange: oodt/trunk/filemgr/src/test/org/apache/oodt/cas/filemgr/datatransfer/TestS3DataTransferer.java
------------------------------------------------------------------------------
svn:mime-type = text/plain