You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/02/17 20:14:18 UTC
incubator-usergrid git commit: Fix ExportServiceIT,
move to JClouds 1.8.0 because old JClouds does not work with Guava 18.
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-273 eba86eeaf -> 32d52c348
Fix ExportServiceIT, move to JClouds 1.8.0 because old JClouds does not work with Guava 18.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/32d52c34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/32d52c34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/32d52c34
Branch: refs/heads/USERGRID-273
Commit: 32d52c348abb2f9588bd6c7db2ccc2d5750359f8
Parents: eba86ee
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Feb 17 14:14:01 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Feb 17 14:14:01 2015 -0500
----------------------------------------------------------------------
stack/pom.xml | 2 +-
.../management/export/S3ExportImpl.java | 6 +-
.../management/importer/S3ImportImpl.java | 28 ++-
.../services/assets/data/BinaryStore.java | 3 +-
.../services/assets/data/S3BinaryStore.java | 18 +-
.../org/apache/usergrid/ServiceITSetupImpl.java | 1 +
.../management/export/ExportServiceIT.java | 183 ++++++++++++-------
.../management/importer/ImportCollectionIT.java | 40 ++--
.../usergrid/management/importer/S3Upload.java | 18 +-
.../src/test/resources/project.properties | 1 +
10 files changed, 193 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 3367317..60cdef4 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -102,7 +102,7 @@
<hector-test-version>1.1-4</hector-test-version>
<jackson-version>1.9.9</jackson-version>
<jackson-2-version>2.3.3</jackson-2-version>
- <jclouds.version>1.6.2-incubating</jclouds.version>
+ <jclouds.version>1.8.0</jclouds.version>
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 006d838..4fff7c7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -19,6 +19,8 @@ package org.apache.usergrid.management.export;
import com.amazonaws.SDKGlobalConfiguration;
import com.google.common.collect.ImmutableSet;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
import com.google.inject.Module;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
@@ -81,8 +83,8 @@ public class S3ExportImpl implements S3Export {
try {
BlobStore blobStore = context.getBlobStore();
BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
- .payload(ephemeral)
- .calculateMD5()
+ .payload( ephemeral )
+ .contentMD5(Files.hash( ephemeral, Hashing.md5() ))
.contentType("application/json");
Blob blob = blobBuilder.build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
index f10eea9..d147fd7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.management.importer;
import com.amazonaws.SDKGlobalConfiguration;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.lucene.document.StringField;
import org.jclouds.ContextBuilder;
@@ -36,10 +37,8 @@ import org.jclouds.netty.config.NettyPayloadModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
+import java.io.*;
+import java.nio.file.Files;
import java.util.*;
@@ -75,13 +74,22 @@ public class S3ImportImpl implements S3Import {
throw new RuntimeException(
"Blob file name " + blobFileName + " not found in bucket " + bucketName );
}
- File tempFile = File.createTempFile( bucketName, RandomStringUtils.randomAlphabetic(10));
- FileOutputStream fop = new FileOutputStream(tempFile);
- blob.getPayload().writeTo(fop);
- fop.close();
- tempFile.deleteOnExit();
- return tempFile;
+ FileOutputStream fop = null;
+ File tempFile;
+ try {
+ tempFile = File.createTempFile(bucketName, RandomStringUtils.randomAlphabetic(10));
+ tempFile.deleteOnExit();
+ fop = new FileOutputStream(tempFile);
+ InputStream is = blob.getPayload().openStream();
+ IOUtils.copyLarge(is, fop);
+ return tempFile;
+
+ } finally {
+ if ( fop != null ) {
+ fop.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
index 904cedb..5ed2918 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.services.assets.data;
import java.io.IOException;
import java.io.InputStream;
+import java.security.NoSuchAlgorithmException;
import java.util.UUID;
import org.apache.usergrid.persistence.Entity;
@@ -30,7 +31,7 @@ public interface BinaryStore {
* writes the inputStream to the store and updates the entity's file-metadata field. however, it doesn't persistent
* the entity.
*/
- void write( UUID appId, Entity entity, InputStream inputStream ) throws IOException;
+ void write( UUID appId, Entity entity, InputStream inputStream ) throws IOException, NoSuchAlgorithmException;
/** read the entity's file data from the store */
InputStream read( UUID appId, Entity entity ) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
index e1748d3..90bd24f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
@@ -24,11 +24,17 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.AsyncBlobStore;
import org.jclouds.blobstore.BlobStore;
@@ -117,8 +123,10 @@ public class S3BinaryStore implements BinaryStore {
if ( written < FIVE_MB ) { // total smaller than 5mb
BlobStore blobStore = getContext().getBlobStore();
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( data ).calculateMD5().contentType( mimeType );
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder(uploadFileName)
+ .payload(data)
+ .contentMD5(Hashing.md5().newHasher().putBytes( data ).hash())
+ .contentType(mimeType);
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
@@ -149,8 +157,10 @@ public class S3BinaryStore implements BinaryStore {
IOUtils.closeQuietly( os );
}
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( tempFile ).calculateMD5().contentType( mimeType );
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+ .payload(tempFile)
+ .contentMD5(Files.hash(tempFile, Hashing.md5()))
+ .contentType(mimeType);
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
index c466080..cb7040b 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
@@ -66,6 +66,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
properties = springResource.getBean( "properties", Properties.class );
smf = springResource.getBean( ServiceManagerFactory.class );
exportService = springResource.getBean( ExportService.class );
+ importService = springResource.getBean( ImportService.class );
//set our security manager for shiro
SecurityUtils.setSecurityManager(springResource.getBean( org.apache.shiro.mgt.SecurityManager.class ));
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index 852f2d1..3a6d8f9 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -22,6 +22,8 @@ import java.io.File;
import java.io.FileReader;
import java.util.*;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Service;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
@@ -33,8 +35,6 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
import org.junit.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +43,6 @@ import org.apache.usergrid.NewOrgAppAdminRule;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.batch.JobExecution;
-import org.apache.usergrid.cassandra.SpringResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.management.ApplicationInfo;
@@ -53,7 +52,6 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.entities.JobData;
-import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.setup.ConcurrentProcessSingleton;
import com.amazonaws.SDKGlobalConfiguration;
@@ -103,10 +101,15 @@ public class ExportServiceIT {
logger.info("in setup");
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
- if ( jobScheduler.state() != Service.State.RUNNING ) {
- jobScheduler.startAsync();
- jobScheduler.awaitRunning();
+ try {
+
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean(JobSchedulerService.class);
+ if (jobScheduler.state() != Service.State.RUNNING) {
+ jobScheduler.startAsync();
+ jobScheduler.awaitRunning();
+ }
+ } catch ( Exception e ) {
+ logger.warn("Ignoring error starting jobScheduler, already started?", e);
}
adminUser = newOrgAppAdminRule.getAdminInfo();
@@ -194,30 +197,36 @@ public class ExportServiceIT {
UUID exportUUID = exportService.schedule( payload );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- //assertEquals(2, a.size() );
+ ObjectMapper mapper = new ObjectMapper();
+ HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- for ( indexCon = 0; indexCon < a.size(); indexCon++ ) {
- JSONObject jObj = ( JSONObject ) a.get( indexCon );
- JSONObject data = ( JSONObject ) jObj.get( "Metadata" );
- String uuid = ( String ) data.get( "uuid" );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ List usersList = (List)collectionsMap.get("users");
+
+ int indexApp = 0;
+ for ( indexApp = 0; indexApp < usersList.size(); indexApp++ ) {
+ Map user = (Map)usersList.get( indexApp );
+ Map userProps = (Map)user.get("Metadata");
+ String uuid = ( String ) userProps.get( "uuid" );
if ( entity[0].getUuid().toString().equals( uuid ) ) {
break;
}
}
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( indexCon );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
-
- assertNotNull( objConnections );
+ assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size());
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
+ Map userMap = (Map)usersList.get( indexApp );
+ Map connectionsMap = (Map)userMap.get("connections");
+ assertNotNull( connectionsMap );
- assertNotNull( objVibrations );
+ List vibrationsList = (List)connectionsMap.get( "Vibrations" );
+ assertNotNull( vibrationsList );
+ f.deleteOnExit();
}
@@ -279,31 +288,34 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- int indexApp = 0;
+ ObjectMapper mapper = new ObjectMapper();
+ HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
+
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ List usersList = (List)collectionsMap.get("users");
- for ( indexApp = 0; indexApp < a.size(); indexApp++ ) {
- JSONObject jObj = ( JSONObject ) a.get( indexApp );
- JSONObject data = ( JSONObject ) jObj.get( "Metadata" );
- String uuid = ( String ) data.get( "uuid" );
+ int indexApp = 0;
+ for ( indexApp = 0; indexApp < usersList.size(); indexApp++ ) {
+ Map user = (Map)usersList.get( indexApp );
+ Map userProps = (Map)user.get("Metadata");
+ String uuid = ( String ) userProps.get( "uuid" );
if ( entity[0].getUuid().toString().equals( uuid ) ) {
break;
}
}
- assertTrue( "Uuid was not found in exported files. ", indexApp < a.size() );
-
+ assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size());
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( indexApp );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
+ Map userMap = (Map)usersList.get( indexApp );
+ Map connectionsMap = (Map)userMap.get("connections");
+ assertNotNull( connectionsMap );
- assertNotNull( objConnections );
+ List vibrationsList = (List)connectionsMap.get( "Vibrations" );
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
-
- assertNotNull( objVibrations );
+ assertNotNull( vibrationsList );
f.deleteOnExit();
}
@@ -351,16 +363,20 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get(collectionName);
- assertEquals( 3, a.size() );
- for ( int i = 0; i < a.size(); i++ ) {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- String entityName = ( String ) entityData.get( "name" );
- // assertNotEquals( "NotEqual","junkRealName",entityName );
+ for ( Object o : collection ) {
+ Map entityMap = (Map)o;
+ Map metadataMap = (Map)entityMap.get("Metadata");
+ String entityName = (String)metadataMap.get("name");
assertFalse( "junkRealName".equals( entityName ) );
}
f.deleteOnExit();
@@ -422,15 +438,20 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get(collectionName);
- //assertEquals( 3 , a.size() );
- for ( int i = 0; i < a.size(); i++ ) {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String ) entityData.get( "name" );
+ for ( Object o : collection ) {
+ Map entityMap = (Map)o;
+ Map metadataMap = (Map)entityMap.get("Metadata");
+ String entityName = (String)metadataMap.get("name");
assertFalse( "junkRealName".equals( entityName ) );
}
}
@@ -485,14 +506,20 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
+
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get( collectionName );
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- assertEquals( 1, a.size() );
- for ( int i = 0; i < a.size(); i++ ) {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String ) entityData.get( "name" );
+ for ( Object o : collection ) {
+ Map entityMap = (Map)o;
+ Map metadataMap = (Map)entityMap.get("Metadata");
+ String entityName = (String)metadataMap.get("name");
assertFalse( "junkRealName".equals( entityName ) );
}
}
@@ -515,6 +542,7 @@ public class ExportServiceIT {
f.deleteOnExit();
EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
// em.createApplicationCollection( "qtsMagics" );
// intialize user object to be posted
Map<String, Object> userProperties = null;
@@ -549,11 +577,17 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get( collectionName );
- assertEquals( entitiesToCreate, a.size() );
+ assertEquals(entitiesToCreate, collection.size());
}
@@ -611,16 +645,22 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collectionList = (List)collectionsMap.get( collectionName );
- // only one entity should match the query.
- assertEquals( 1, a.size() );
+ assertEquals(1, collectionList.size());
}
@Test
+ @Ignore("this is a meaningless test because our export format does not support export of organizations")
public void testExportOneOrganization() throws Exception {
// create a bunch of organizations, each with applications and collections of entities
@@ -681,11 +721,16 @@ public class ExportServiceIT {
File exportedFile = new File( exportFileName );
exportedFile.deleteOnExit();
- JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray )
- parser.parse( new FileReader( exportedFile ) );
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( exportedFile ), typeRef);
+ Map collectionsMap = (Map)jsonMap.get("collections");
+
+ List collectionList = (List)collectionsMap.get("users");
- assertEquals( 23, a.size() );
+ assertEquals( 3, collectionList.size() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index e566512..c2e3161 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -18,6 +18,7 @@
package org.apache.usergrid.management.importer;
+import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -97,8 +98,7 @@ public class ImportCollectionIT {
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
@ClassRule
- public static final ServiceITSetup setup =
- new ServiceITSetupImpl( );
+ public static final ServiceITSetup setup = new ServiceITSetupImpl( );
@Rule
public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
@@ -110,7 +110,9 @@ public class ImportCollectionIT {
bucketPrefix = System.getProperty( "bucketName" );
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton
+ .getInstance().getSpringResource().getBean( JobSchedulerService.class );
+
if ( jobScheduler.state() != Service.State.RUNNING ) {
jobScheduler.startAsync();
jobScheduler.awaitRunning();
@@ -366,8 +368,12 @@ public class ImportCollectionIT {
// export and upload a bad JSON file to the S3 bucket
+ File cwd = new File(".");
+ String basePath = System.getProperty("target.directory")
+ + File.separator + "test-classes" + File.separator;
+
List<String> filenames = new ArrayList<>( 1 );
- filenames.add( "testimport-bad-json.json");
+ filenames.add( basePath + "testimport-bad-json.json");
S3Upload s3Upload = new S3Upload();
s3Upload.copyToS3(
@@ -412,11 +418,16 @@ public class ImportCollectionIT {
// upload good and badly formatted files to our S3 bucket
+ File cwd = new File(".");
+ String basePath = cwd.getAbsolutePath();
+ basePath = basePath.substring( 0 , basePath.length() - 1 );
+ basePath = basePath + "src" + File.separator + "test" + File.separator + "resource" + File.separator;
+
List<String> filenames = new ArrayList<>( 3 );
- filenames.add( "testimport-with-connections.json" );
- filenames.add( "testimport-qtmagics.json" );
- filenames.add( "testimport-bad-connection.json" );
- filenames.add( "testimport-bad-json.json" );
+ filenames.add( basePath + "testimport-with-connections.json" );
+ filenames.add( basePath + "testimport-qtmagics.json" );
+ filenames.add( basePath + "testimport-bad-connection.json" );
+ filenames.add( basePath + "testimport-bad-json.json" );
S3Upload s3Upload = new S3Upload();
s3Upload.copyToS3(
@@ -481,11 +492,12 @@ public class ImportCollectionIT {
ImportService importService = setup.getImportService();
- Import importEntity = importService.schedule(em.getApplication().getUuid(), new HashMap<String, Object>() {{
- put( "path", organization.getName() + em.getApplication().getName() );
- put( "organizationId", organization.getUuid() );
- put( "applicationId", em.getApplication().getUuid() );
- put( "properties", new HashMap<String, Object>() {{
+ Import importEntity = importService.schedule(em.getApplication().getUuid(),
+ new HashMap<String, Object>() {{
+ put( "path", organization.getName() + em.getApplication().getName() );
+ put( "organizationId", organization.getUuid() );
+ put( "applicationId", em.getApplication().getUuid() );
+ put( "properties", new HashMap<String, Object>() {{
put( "storage_provider", "s3" );
put( "storage_info", new HashMap<String, Object>() {{
put( "s3_access_id",
@@ -495,7 +507,7 @@ public class ImportCollectionIT {
put( "bucket_location", bucketName );
}} );
}} );
- }} );
+ }});
int maxRetries = 30;
int retries = 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
index 5b639c2..c56ff66 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
@@ -17,12 +17,17 @@
package org.apache.usergrid.management.importer;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
@@ -47,7 +52,8 @@ public class S3Upload {
private static final Logger logger = LoggerFactory.getLogger( S3Upload.class );
- public void copyToS3( String accessKey, String secretKey, String bucketName, List<String> filenames ) {
+ public void copyToS3( String accessKey, String secretKey, String bucketName, List<String> filenames )
+ throws FileNotFoundException {
Properties overrides = new Properties();
overrides.setProperty( "s3" + ".identity", accessKey );
@@ -77,16 +83,16 @@ public class S3Upload {
Iterator<String> fileNameIterator = filenames.iterator();
- while(fileNameIterator.hasNext()) {
+ while (fileNameIterator.hasNext()) {
String filename = fileNameIterator.next();
- InputStream fileStream = S3Upload.class.getResourceAsStream( "/" + filename );
-
+ File uploadFile = new File( filename );
+
try {
BlobStore blobStore = context.getBlobStore();
BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
- .payload( fileStream )
- .calculateMD5()
+ .payload( uploadFile )
+ .contentMD5(Files.hash( uploadFile, Hashing.md5()))
.contentType( "application/json" );
Blob blob = blobBuilder.build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/project.properties b/stack/services/src/test/resources/project.properties
index 94ef3bd..d38e878 100644
--- a/stack/services/src/test/resources/project.properties
+++ b/stack/services/src/test/resources/project.properties
@@ -15,4 +15,5 @@
# limitations under the License.
target.directory=${project.build.directory}
+resources.dir=${project.build.directory}
jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar