You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/25 17:21:24 UTC
[30/51] [abbrv] git commit: Added amazon s3 dependencies to get the
file back out of s3 to make sure it was transferred over. Removed smoke from
smoke tests,
they now properly get the file back from s3. Json validity is handled in other
tests so I only v
Added amazon s3 dependencies to get the file back out of s3 to make sure it was transferred over.
Removed smoke from smoke tests, they now properly get the file back from s3. Json validity is handled in other tests so I only verify that the object exists.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a1b6124c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a1b6124c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a1b6124c
Branch: refs/heads/two-dot-o
Commit: a1b6124ccf6aed8de351693f184302381a777994
Parents: fe2b3d6
Author: grey <gr...@apigee.com>
Authored: Wed Mar 19 16:18:15 2014 -0700
Committer: grey <gr...@apigee.com>
Committed: Wed Mar 19 16:18:15 2014 -0700
----------------------------------------------------------------------
stack/services/pom.xml | 13 +++++++
.../management/export/S3ExportImpl.java | 10 +++--
.../cassandra/ManagementServiceIT.java | 39 +++++++++++++++++++-
.../management/cassandra/MockS3ExportImpl.java | 1 -
4 files changed, 58 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b6124c/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 5c31d43..de02659 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -553,5 +553,18 @@
<artifactId>jul-to-slf4j</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>4.3.3</version>
+ </dependency>
+
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk</artifactId>
+ <version>1.7.3</version>
+ </dependency>
+
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b6124c/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 863a8f8..c6cfc37 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -44,9 +44,12 @@ import com.google.inject.Module;
*/
public class S3ExportImpl implements S3Export {
+ String fn;
+
@Override
public void copyToS3( final InputStream inputStream, final Map<String,Object> exportInfo, String filename ) {
+ fn = filename;
Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
/*won't need any of the properties as I have the export info*/
@@ -87,13 +90,14 @@ public class S3ExportImpl implements S3Export {
try {
AsyncBlobStore blobStore = context.getAsyncBlobStore();
BlobBuilder blobBuilder =
- blobStore.blobBuilder( filename ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+ blobStore.blobBuilder( fn ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
Blob blob = blobBuilder.build();
ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+
logger.info( "Uploaded file etag=" + futureETag.get() );
}
catch ( Exception e ) {
@@ -102,8 +106,8 @@ public class S3ExportImpl implements S3Export {
}
@Override
- public String getFilename () {return "";}
+ public String getFilename () {return fn;}
@Override
- public void setFilename(String givenName) {;}
+ public void setFilename(String givenName) {fn = givenName;}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b6124c/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index 6c0431b..310d46a 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -65,6 +65,13 @@ import org.apache.usergrid.security.tokens.exceptions.InvalidTokenException;
import org.apache.usergrid.utils.JsonUtils;
import org.apache.usergrid.utils.UUIDUtils;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+
import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
@@ -1473,6 +1480,9 @@ public class ManagementServiceIT {
@Test //For this test please input your s3 credentials into payload builder.
public void testIntegration100EntitiesOn() throws Exception {
+ //s3client.putObject(new PutObjectRequest(bucketName, keyName, file));
+
+
S3Export s3Export = new S3ExportImpl();
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
@@ -1507,9 +1517,24 @@ public class ManagementServiceIT {
when( jobExecution.getJobData() ).thenReturn( jobData );
exportService.doExport( jobExecution );
+ Thread.sleep(1000);
+
+ AWSCredentials myCredentials = new BasicAWSCredentials(
+ System.getProperty( "accessKey" ), System.getProperty("secretKey"));
+ AmazonS3 s3client = new AmazonS3Client(myCredentials);
+ S3Object sd = null;
+ try {
+ sd = s3client.getObject( new GetObjectRequest( System.getProperty( "bucketName" ),
+ s3Export.getFilename()) );
+ }catch(Exception e) {
+ assert(false);
+ }
+ assertNotNull( sd );
+
}
- @Test //For this test please input your s3 credentials into payload builder.
+ //@Ignore("I'd have to add some overhead such that I loop through all files or keep track of them all ")
+ @Test
public void testIntegration100EntitiesOnOneOrg() throws Exception {
S3Export s3Export = new S3ExportImpl();
@@ -1567,6 +1592,18 @@ public class ManagementServiceIT {
when( jobExecution.getJobData() ).thenReturn( jobData );
exportService.doExport( jobExecution );
+
+ AWSCredentials myCredentials = new BasicAWSCredentials(
+ System.getProperty( "accessKey" ), System.getProperty("secretKey"));
+ AmazonS3 s3client = new AmazonS3Client(myCredentials);
+ S3Object sd = null;
+ try {
+ sd = s3client.getObject( new GetObjectRequest( System.getProperty( "bucketName" ),
+ s3Export.getFilename()) );
+ }catch(Exception e) {
+ assert(false);
+ }
+ assertNotNull( sd );
}
/*Creates fake payload for testing purposes.*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a1b6124c/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index 8ef1c19..a1814fd 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -42,7 +42,6 @@ public class MockS3ExportImpl implements S3Export {
int read = 0;
byte[] bytes = new byte[1024];
OutputStream outputStream = null;
- //FileInputStream fis = new PrintWriter( inputStream );
try {
outputStream = new FileOutputStream( new File( getFilename() ) );