You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/02/21 15:02:12 UTC
[06/25] incubator-usergrid git commit: Fix ExportServiceIT,
move to JClouds 1.8.0 because old JClouds does not work with Guava 18.
Fix ExportServiceIT, move to JClouds 1.8.0 because old JClouds does not work with Guava 18.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/32d52c34
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/32d52c34
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/32d52c34
Branch: refs/heads/app-rebuild-fix
Commit: 32d52c348abb2f9588bd6c7db2ccc2d5750359f8
Parents: eba86ee
Author: Dave Johnson <dm...@apigee.com>
Authored: Tue Feb 17 14:14:01 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Tue Feb 17 14:14:01 2015 -0500
----------------------------------------------------------------------
stack/pom.xml | 2 +-
.../management/export/S3ExportImpl.java | 6 +-
.../management/importer/S3ImportImpl.java | 28 ++-
.../services/assets/data/BinaryStore.java | 3 +-
.../services/assets/data/S3BinaryStore.java | 18 +-
.../org/apache/usergrid/ServiceITSetupImpl.java | 1 +
.../management/export/ExportServiceIT.java | 183 ++++++++++++-------
.../management/importer/ImportCollectionIT.java | 40 ++--
.../usergrid/management/importer/S3Upload.java | 18 +-
.../src/test/resources/project.properties | 1 +
10 files changed, 193 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 3367317..60cdef4 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -102,7 +102,7 @@
<hector-test-version>1.1-4</hector-test-version>
<jackson-version>1.9.9</jackson-version>
<jackson-2-version>2.3.3</jackson-2-version>
- <jclouds.version>1.6.2-incubating</jclouds.version>
+ <jclouds.version>1.8.0</jclouds.version>
<jersey-version>1.18.1</jersey-version>
<junit-version>4.12</junit-version>
<log4j-version>1.2.16</log4j-version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 006d838..4fff7c7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -19,6 +19,8 @@ package org.apache.usergrid.management.export;
import com.amazonaws.SDKGlobalConfiguration;
import com.google.common.collect.ImmutableSet;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
import com.google.inject.Module;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
@@ -81,8 +83,8 @@ public class S3ExportImpl implements S3Export {
try {
BlobStore blobStore = context.getBlobStore();
BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
- .payload(ephemeral)
- .calculateMD5()
+ .payload( ephemeral )
+ .contentMD5(Files.hash( ephemeral, Hashing.md5() ))
.contentType("application/json");
Blob blob = blobBuilder.build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
index f10eea9..d147fd7 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/S3ImportImpl.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.management.importer;
import com.amazonaws.SDKGlobalConfiguration;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Module;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.lucene.document.StringField;
import org.jclouds.ContextBuilder;
@@ -36,10 +37,8 @@ import org.jclouds.netty.config.NettyPayloadModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
+import java.io.*;
+import java.nio.file.Files;
import java.util.*;
@@ -75,13 +74,22 @@ public class S3ImportImpl implements S3Import {
throw new RuntimeException(
"Blob file name " + blobFileName + " not found in bucket " + bucketName );
}
- File tempFile = File.createTempFile( bucketName, RandomStringUtils.randomAlphabetic(10));
- FileOutputStream fop = new FileOutputStream(tempFile);
- blob.getPayload().writeTo(fop);
- fop.close();
- tempFile.deleteOnExit();
- return tempFile;
+ FileOutputStream fop = null;
+ File tempFile;
+ try {
+ tempFile = File.createTempFile(bucketName, RandomStringUtils.randomAlphabetic(10));
+ tempFile.deleteOnExit();
+ fop = new FileOutputStream(tempFile);
+ InputStream is = blob.getPayload().openStream();
+ IOUtils.copyLarge(is, fop);
+ return tempFile;
+
+ } finally {
+ if ( fop != null ) {
+ fop.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
index 904cedb..5ed2918 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/BinaryStore.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.services.assets.data;
import java.io.IOException;
import java.io.InputStream;
+import java.security.NoSuchAlgorithmException;
import java.util.UUID;
import org.apache.usergrid.persistence.Entity;
@@ -30,7 +31,7 @@ public interface BinaryStore {
* writes the inputStream to the store and updates the entity's file-metadata field. however, it doesn't persistent
* the entity.
*/
- void write( UUID appId, Entity entity, InputStream inputStream ) throws IOException;
+ void write( UUID appId, Entity entity, InputStream inputStream ) throws IOException, NoSuchAlgorithmException;
/** read the entity's file data from the store */
InputStream read( UUID appId, Entity entity ) throws IOException;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
index e1748d3..90bd24f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/assets/data/S3BinaryStore.java
@@ -24,11 +24,17 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import com.google.common.hash.HashCode;
+import com.google.common.hash.HashFunction;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.AsyncBlobStore;
import org.jclouds.blobstore.BlobStore;
@@ -117,8 +123,10 @@ public class S3BinaryStore implements BinaryStore {
if ( written < FIVE_MB ) { // total smaller than 5mb
BlobStore blobStore = getContext().getBlobStore();
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( data ).calculateMD5().contentType( mimeType );
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder(uploadFileName)
+ .payload(data)
+ .contentMD5(Hashing.md5().newHasher().putBytes( data ).hash())
+ .contentType(mimeType);
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
@@ -149,8 +157,10 @@ public class S3BinaryStore implements BinaryStore {
IOUtils.closeQuietly( os );
}
- BlobBuilder.PayloadBlobBuilder bb =
- blobStore.blobBuilder( uploadFileName ).payload( tempFile ).calculateMD5().contentType( mimeType );
+ BlobBuilder.PayloadBlobBuilder bb = blobStore.blobBuilder( uploadFileName )
+ .payload(tempFile)
+ .contentMD5(Files.hash(tempFile, Hashing.md5()))
+ .contentType(mimeType);
fileMetadata.put( AssetUtils.CONTENT_LENGTH, written );
if ( fileMetadata.get( AssetUtils.CONTENT_DISPOSITION ) != null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
index c466080..cb7040b 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
@@ -66,6 +66,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
properties = springResource.getBean( "properties", Properties.class );
smf = springResource.getBean( ServiceManagerFactory.class );
exportService = springResource.getBean( ExportService.class );
+ importService = springResource.getBean( ImportService.class );
//set our security manager for shiro
SecurityUtils.setSecurityManager(springResource.getBean( org.apache.shiro.mgt.SecurityManager.class ));
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index 852f2d1..3a6d8f9 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@ -22,6 +22,8 @@ import java.io.File;
import java.io.FileReader;
import java.util.*;
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.util.concurrent.Service;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.lang3.StringUtils;
@@ -33,8 +35,6 @@ import org.jclouds.blobstore.domain.Blob;
import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
import org.junit.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,7 +43,6 @@ import org.apache.usergrid.NewOrgAppAdminRule;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.batch.JobExecution;
-import org.apache.usergrid.cassandra.SpringResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.management.ApplicationInfo;
@@ -53,7 +52,6 @@ import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.entities.JobData;
-import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.setup.ConcurrentProcessSingleton;
import com.amazonaws.SDKGlobalConfiguration;
@@ -103,10 +101,15 @@ public class ExportServiceIT {
logger.info("in setup");
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
- if ( jobScheduler.state() != Service.State.RUNNING ) {
- jobScheduler.startAsync();
- jobScheduler.awaitRunning();
+ try {
+
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean(JobSchedulerService.class);
+ if (jobScheduler.state() != Service.State.RUNNING) {
+ jobScheduler.startAsync();
+ jobScheduler.awaitRunning();
+ }
+ } catch ( Exception e ) {
+ logger.warn("Ignoring error starting jobScheduler, already started?", e);
}
adminUser = newOrgAppAdminRule.getAdminInfo();
@@ -194,30 +197,36 @@ public class ExportServiceIT {
UUID exportUUID = exportService.schedule( payload );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- //assertEquals(2, a.size() );
+ ObjectMapper mapper = new ObjectMapper();
+ HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- for ( indexCon = 0; indexCon < a.size(); indexCon++ ) {
- JSONObject jObj = ( JSONObject ) a.get( indexCon );
- JSONObject data = ( JSONObject ) jObj.get( "Metadata" );
- String uuid = ( String ) data.get( "uuid" );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ List usersList = (List)collectionsMap.get("users");
+
+ int indexApp = 0;
+ for ( indexApp = 0; indexApp < usersList.size(); indexApp++ ) {
+ Map user = (Map)usersList.get( indexApp );
+ Map userProps = (Map)user.get("Metadata");
+ String uuid = ( String ) userProps.get( "uuid" );
if ( entity[0].getUuid().toString().equals( uuid ) ) {
break;
}
}
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( indexCon );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
-
- assertNotNull( objConnections );
+ assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size());
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
+ Map userMap = (Map)usersList.get( indexApp );
+ Map connectionsMap = (Map)userMap.get("connections");
+ assertNotNull( connectionsMap );
- assertNotNull( objVibrations );
+ List vibrationsList = (List)connectionsMap.get( "Vibrations" );
+ assertNotNull( vibrationsList );
+ f.deleteOnExit();
}
@@ -279,31 +288,34 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- int indexApp = 0;
+ ObjectMapper mapper = new ObjectMapper();
+ HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
+
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ List usersList = (List)collectionsMap.get("users");
- for ( indexApp = 0; indexApp < a.size(); indexApp++ ) {
- JSONObject jObj = ( JSONObject ) a.get( indexApp );
- JSONObject data = ( JSONObject ) jObj.get( "Metadata" );
- String uuid = ( String ) data.get( "uuid" );
+ int indexApp = 0;
+ for ( indexApp = 0; indexApp < usersList.size(); indexApp++ ) {
+ Map user = (Map)usersList.get( indexApp );
+ Map userProps = (Map)user.get("Metadata");
+ String uuid = ( String ) userProps.get( "uuid" );
if ( entity[0].getUuid().toString().equals( uuid ) ) {
break;
}
}
- assertTrue( "Uuid was not found in exported files. ", indexApp < a.size() );
-
+ assertTrue("Uuid was not found in exported files. ", indexApp < usersList.size());
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( indexApp );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
+ Map userMap = (Map)usersList.get( indexApp );
+ Map connectionsMap = (Map)userMap.get("connections");
+ assertNotNull( connectionsMap );
- assertNotNull( objConnections );
+ List vibrationsList = (List)connectionsMap.get( "Vibrations" );
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
-
- assertNotNull( objVibrations );
+ assertNotNull( vibrationsList );
f.deleteOnExit();
}
@@ -351,16 +363,20 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get(collectionName);
- assertEquals( 3, a.size() );
- for ( int i = 0; i < a.size(); i++ ) {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- String entityName = ( String ) entityData.get( "name" );
- // assertNotEquals( "NotEqual","junkRealName",entityName );
+ for ( Object o : collection ) {
+ Map entityMap = (Map)o;
+ Map metadataMap = (Map)entityMap.get("Metadata");
+ String entityName = (String)metadataMap.get("name");
assertFalse( "junkRealName".equals( entityName ) );
}
f.deleteOnExit();
@@ -422,15 +438,20 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get(collectionName);
- //assertEquals( 3 , a.size() );
- for ( int i = 0; i < a.size(); i++ ) {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String ) entityData.get( "name" );
+ for ( Object o : collection ) {
+ Map entityMap = (Map)o;
+ Map metadataMap = (Map)entityMap.get("Metadata");
+ String entityName = (String)metadataMap.get("name");
assertFalse( "junkRealName".equals( entityName ) );
}
}
@@ -485,14 +506,20 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
+
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get( collectionName );
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- assertEquals( 1, a.size() );
- for ( int i = 0; i < a.size(); i++ ) {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String ) entityData.get( "name" );
+ for ( Object o : collection ) {
+ Map entityMap = (Map)o;
+ Map metadataMap = (Map)entityMap.get("Metadata");
+ String entityName = (String)metadataMap.get("name");
assertFalse( "junkRealName".equals( entityName ) );
}
}
@@ -515,6 +542,7 @@ public class ExportServiceIT {
f.deleteOnExit();
EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
// em.createApplicationCollection( "qtsMagics" );
// intialize user object to be posted
Map<String, Object> userProperties = null;
@@ -549,11 +577,17 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ HashMap<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collection = (List)collectionsMap.get( collectionName );
- assertEquals( entitiesToCreate, a.size() );
+ assertEquals(entitiesToCreate, collection.size());
}
@@ -611,16 +645,22 @@ public class ExportServiceIT {
exportService.doExport( jobExecution );
- JSONParser parser = new JSONParser();
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( f ), typeRef);
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ Map collectionsMap = (Map)jsonMap.get("collections");
+ String collectionName = (String)collectionsMap.keySet().iterator().next();
+ List collectionList = (List)collectionsMap.get( collectionName );
- // only one entity should match the query.
- assertEquals( 1, a.size() );
+ assertEquals(1, collectionList.size());
}
@Test
+ @Ignore("this is a meaningless test because our export format does not support export of organizations")
public void testExportOneOrganization() throws Exception {
// create a bunch of organizations, each with applications and collections of entities
@@ -681,11 +721,16 @@ public class ExportServiceIT {
File exportedFile = new File( exportFileName );
exportedFile.deleteOnExit();
- JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray )
- parser.parse( new FileReader( exportedFile ) );
+ TypeReference<HashMap<String,Object>> typeRef
+ = new TypeReference<HashMap<String,Object>>() {};
+
+ ObjectMapper mapper = new ObjectMapper();
+ Map<String,Object> jsonMap = mapper.readValue(new FileReader( exportedFile ), typeRef);
+ Map collectionsMap = (Map)jsonMap.get("collections");
+
+ List collectionList = (List)collectionsMap.get("users");
- assertEquals( 23, a.size() );
+ assertEquals( 3, collectionList.size() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index e566512..c2e3161 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -18,6 +18,7 @@
package org.apache.usergrid.management.importer;
+import java.io.File;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -97,8 +98,7 @@ public class ImportCollectionIT {
public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
@ClassRule
- public static final ServiceITSetup setup =
- new ServiceITSetupImpl( );
+ public static final ServiceITSetup setup = new ServiceITSetupImpl( );
@Rule
public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
@@ -110,7 +110,9 @@ public class ImportCollectionIT {
bucketPrefix = System.getProperty( "bucketName" );
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton
+ .getInstance().getSpringResource().getBean( JobSchedulerService.class );
+
if ( jobScheduler.state() != Service.State.RUNNING ) {
jobScheduler.startAsync();
jobScheduler.awaitRunning();
@@ -366,8 +368,12 @@ public class ImportCollectionIT {
// export and upload a bad JSON file to the S3 bucket
+ File cwd = new File(".");
+ String basePath = System.getProperty("target.directory")
+ + File.separator + "test-classes" + File.separator;
+
List<String> filenames = new ArrayList<>( 1 );
- filenames.add( "testimport-bad-json.json");
+ filenames.add( basePath + "testimport-bad-json.json");
S3Upload s3Upload = new S3Upload();
s3Upload.copyToS3(
@@ -412,11 +418,16 @@ public class ImportCollectionIT {
// upload good and badly formatted files to our S3 bucket
+ File cwd = new File(".");
+ String basePath = cwd.getAbsolutePath();
+ basePath = basePath.substring( 0 , basePath.length() - 1 );
+ basePath = basePath + "src" + File.separator + "test" + File.separator + "resource" + File.separator;
+
List<String> filenames = new ArrayList<>( 3 );
- filenames.add( "testimport-with-connections.json" );
- filenames.add( "testimport-qtmagics.json" );
- filenames.add( "testimport-bad-connection.json" );
- filenames.add( "testimport-bad-json.json" );
+ filenames.add( basePath + "testimport-with-connections.json" );
+ filenames.add( basePath + "testimport-qtmagics.json" );
+ filenames.add( basePath + "testimport-bad-connection.json" );
+ filenames.add( basePath + "testimport-bad-json.json" );
S3Upload s3Upload = new S3Upload();
s3Upload.copyToS3(
@@ -481,11 +492,12 @@ public class ImportCollectionIT {
ImportService importService = setup.getImportService();
- Import importEntity = importService.schedule(em.getApplication().getUuid(), new HashMap<String, Object>() {{
- put( "path", organization.getName() + em.getApplication().getName() );
- put( "organizationId", organization.getUuid() );
- put( "applicationId", em.getApplication().getUuid() );
- put( "properties", new HashMap<String, Object>() {{
+ Import importEntity = importService.schedule(em.getApplication().getUuid(),
+ new HashMap<String, Object>() {{
+ put( "path", organization.getName() + em.getApplication().getName() );
+ put( "organizationId", organization.getUuid() );
+ put( "applicationId", em.getApplication().getUuid() );
+ put( "properties", new HashMap<String, Object>() {{
put( "storage_provider", "s3" );
put( "storage_info", new HashMap<String, Object>() {{
put( "s3_access_id",
@@ -495,7 +507,7 @@ public class ImportCollectionIT {
put( "bucket_location", bucketName );
}} );
}} );
- }} );
+ }});
int maxRetries = 30;
int retries = 0;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
index 5b639c2..c56ff66 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/S3Upload.java
@@ -17,12 +17,17 @@
package org.apache.usergrid.management.importer;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.InputStream;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import com.google.common.hash.Hashing;
+import com.google.common.io.Files;
import org.jclouds.ContextBuilder;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.BlobStoreContext;
@@ -47,7 +52,8 @@ public class S3Upload {
private static final Logger logger = LoggerFactory.getLogger( S3Upload.class );
- public void copyToS3( String accessKey, String secretKey, String bucketName, List<String> filenames ) {
+ public void copyToS3( String accessKey, String secretKey, String bucketName, List<String> filenames )
+ throws FileNotFoundException {
Properties overrides = new Properties();
overrides.setProperty( "s3" + ".identity", accessKey );
@@ -77,16 +83,16 @@ public class S3Upload {
Iterator<String> fileNameIterator = filenames.iterator();
- while(fileNameIterator.hasNext()) {
+ while (fileNameIterator.hasNext()) {
String filename = fileNameIterator.next();
- InputStream fileStream = S3Upload.class.getResourceAsStream( "/" + filename );
-
+ File uploadFile = new File( filename );
+
try {
BlobStore blobStore = context.getBlobStore();
BlobBuilder blobBuilder = blobStore.blobBuilder( filename )
- .payload( fileStream )
- .calculateMD5()
+ .payload( uploadFile )
+ .contentMD5(Files.hash( uploadFile, Hashing.md5()))
.contentType( "application/json" );
Blob blob = blobBuilder.build();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/32d52c34/stack/services/src/test/resources/project.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/project.properties b/stack/services/src/test/resources/project.properties
index 94ef3bd..d38e878 100644
--- a/stack/services/src/test/resources/project.properties
+++ b/stack/services/src/test/resources/project.properties
@@ -15,4 +15,5 @@
# limitations under the License.
target.directory=${project.build.directory}
+resources.dir=${project.build.directory}
jamm.path=-javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar