You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/02/25 19:20:49 UTC
[1/9] Added classes to the branch that were labeled untracked.
Repository: incubator-usergrid
Updated Branches:
refs/pull/60/head 22a17dc80 -> 96a1a72df
refs/pull/60/merge cfef069c7 -> 78147cb31 (forced update)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
deleted file mode 100644
index e61d64c..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.io.InputStream;
-
-import org.usergrid.management.ExportInfo;
-
-
-/**
- *
- *
- */
-public interface S3Export {
- void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename );
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
deleted file mode 100644
index c574e01..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.io.InputStream;
-import java.util.Properties;
-
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.AsyncBlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.domain.BlobBuilder;
-import org.jclouds.blobstore.options.PutOptions;
-import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
-import org.jclouds.logging.log4j.config.Log4JLoggingModule;
-import org.jclouds.netty.config.NettyPayloadModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.usergrid.management.ExportInfo;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Module;
-
-
-/**
- *
- *
- */
-public class S3ExportImpl implements S3Export {
-
- @Override
- public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
-
- Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
- /*won't need any of the properties as I have the export info*/
- String bucketName = exportInfo.getBucket_location();
- String accessId = exportInfo.getS3_accessId();
- String secretKey = exportInfo.getS3_key();
-
- Properties overrides = new Properties();
- overrides.setProperty( "s3" + ".identity", accessId );
- overrides.setProperty( "s3" + ".credential", secretKey );
-
- final Iterable<? extends Module> MODULES = ImmutableSet
- .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule
- () );
-
- BlobStoreContext context =
- ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
- .overrides( overrides ).buildView( BlobStoreContext.class );
-
- // Create Container (the bucket in s3)
- try {
- AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
- // BlobStore (returns false if it already exists)
- ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName );
- if ( container.get() ) {
- logger.info( "Created bucket " + bucketName );
- }
- }
- catch ( Exception ex ) {
- logger.error( "Could not start binary service: {}", ex.getMessage() );
- return;
- }
-
- try {
- AsyncBlobStore blobStore = context.getAsyncBlobStore();
- BlobBuilder blobBuilder =
- blobStore.blobBuilder( filename ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
-
-
- Blob blob = blobBuilder.build();
-
- ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
-
- logger.info( "Uploaded file etag=" + futureETag.get() );
- }
- catch ( Exception e ) {
- logger.error( "Error uploading to blob store", e );
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index d663890..ead98e1 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -65,12 +65,12 @@
<property name="managementService" ref="managementService"/>
</bean>
- <bean id="exportService" class="org.usergrid.management.export.ExportServiceImpl" >
+ <bean id="exportService" class="org.apache.usergrid.management.export.ExportServiceImpl" >
<property name="managementService" ref="managementService"/>
<property name="emf" ref="entityManagerFactory"/>
<property name="sch" ref="schedulerService"/>
</bean>
- <bean id="s3Export" class="org.usergrid.management.export.S3ExportImpl" />
+ <bean id="s3Export" class="org.apache.usergrid.management.export.S3ExportImpl" />
</beans>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
index 12e2ab0..4df8e5e 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
@@ -5,6 +5,7 @@ import java.util.Properties;
import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory;
@@ -21,6 +22,8 @@ public interface ServiceITSetup extends CoreITSetup {
Properties getProps();
+ ExportService getExportService();
+
/**
* Convenience method to set a property in the Properties object returned by getProps();
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
index 6008610..e200b4f 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
@@ -11,6 +11,7 @@ import org.springframework.beans.factory.config.PropertiesFactoryBean;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
@@ -27,6 +28,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
private TokenService tokenService;
private SignInProviderFactory providerFactory;
private Properties properties;
+ private ExportService exportService;
public ServiceITSetupImpl( CassandraResource cassandraResource ) {
@@ -48,6 +50,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
providerFactory = cassandraResource.getBean( SignInProviderFactory.class );
properties = cassandraResource.getBean( PropertiesFactoryBean.class ).getObject();
smf = cassandraResource.getBean( ServiceManagerFactory.class );
+ exportService = cassandraResource.getBean( ExportService.class );
LOG.info( "Test setup complete..." );
}
@@ -82,6 +85,9 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
return managementService;
}
+ @Override
+ public ExportService getExportService() { return exportService; }
+
public ServiceManagerFactory getSmf() {
if ( smf == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index 99b2ee9..e1b297c 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -1,20 +1,35 @@
package org.apache.usergrid.management.cassandra;
+import java.io.File;
+import java.io.FileReader;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.management.export.ExportJob;
+import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.management.export.S3Export;
+import org.apache.usergrid.management.export.S3ExportImpl;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.Export;
+
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.cassandra.Concurrent;
@@ -23,6 +38,7 @@ import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.entities.JobData;
import org.apache.usergrid.persistence.entities.User;
import org.apache.usergrid.security.AuthPrincipalType;
import org.apache.usergrid.security.crypto.command.Md5HashCommand;
@@ -42,6 +58,8 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/** @author zznate */
@@ -726,4 +744,408 @@ public class ManagementServiceIT {
assertEquals( userId, authedUser.getUuid() );
}
+ //Tests to make sure we can call the job with mock data and it runs.
+ @Ignore
+ public void testFileConnections() throws Exception {
+
+ File f = null;
+
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[10];
+ //creates entities
+ for (int i = 0; i< 10;i++) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+
+ entity[i] = em.create( "user", userProperties );
+
+ }
+ //creates connections
+ em.createConnection( em.getRef( entity[0].getUuid() ),"Vibrations",em.getRef( entity[1].getUuid() ) );
+ em.createConnection( em.getRef( entity[1].getUuid() ),"Vibrations",em.getRef( entity[0].getUuid() ) );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ //assertEquals(13, a.size() );
+
+ org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 0 );
+ org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject) objEnt.get( "connections" );
+
+ assertNotNull( objConnections );
+
+ org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get("Vibrations");
+
+ assertNotNull( objVibrations );
+
+ f.delete();
+ }
+
+
+ @Test
+ public void testFileValidity() throws Exception {
+
+ File f = null;
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
+ assertNotNull( entityData );
+
+ }
+ f.delete();
+ }
+
+ @Test
+ public void testFileExportOneOrg() throws Exception {
+
+ File f = null;
+
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ }
+ setup.getMgmtSvc().createOwnerAndOrganization( "noExport","junkUserName","junkRealName","ugExport@usergrid.com","123456789" );
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+
+ //assertEquals( 3 , a.size() );
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
+ String entityName = ( String) entityData.get( "name" );
+ // assertNotEquals( "NotEqual","junkRealName",entityName );
+ assertFalse( "junkRealName".equals( entityName ) );
+
+ }
+ f.delete();
+ }
+ @Test
+ public void testFileExportOneApp() throws Exception {
+
+ File f = null;
+ String orgName = "ed-organization";
+ String appName = "testAppNotExported";
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
+ }
+
+ UUID appId = setup.getEmf().createApplication( orgName, appName );
+
+
+ EntityManager em = setup.getEmf().getEntityManager( appId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[10];
+ //creates entities
+ for (int i = 0; i< 10;i++) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ entity[i] = em.create( "user", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+
+ //assertEquals( 3 , a.size() );
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
+ String entityName = ( String) entityData.get( "name" );
+ assertFalse( "junkRealName".equals( entityName ) );
+ //assertNotEquals( "NotEquals","junkRealName",entityName );
+ }
+ f.delete();
+ }
+
+ //only handles the DoJob Code , different tests for DoExport
+ @Test
+ public void testExportDoJob() throws Exception {
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo ); //this needs to be populated with properties of export info
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ ExportService eS = mock (ExportService.class);
+ job.setExportService( eS );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+
+ }
+
+ @Test
+ public void testExportDoExport() throws Exception {
+
+ EntityManagerFactory emf = setup.getEmf();
+ EntityManager em = emf.getEntityManager( applicationId );
+ HashMap<String, Object> payload = payloadBuilder();
+ ExportService eS = setup.getExportService();
+
+ JobExecution jobExecution = mock( JobExecution.class);
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID entityExportUUID = eS.schedule( exportInfo );
+
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", entityExportUUID);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ //Exportem.get(entityExport);
+ Export exportEntity = ( Export ) em.get(entityExportUUID);
+ assertNotNull( exportEntity );
+ String derp = exportEntity.getState().name();
+ assertEquals( "PENDING",exportEntity.getState().name());
+ try {
+ eS.doExport( exportInfo,jobExecution );
+ }catch(Exception e) {
+ throw e;
+ //assert(false);
+ }
+ exportEntity = ( Export ) em.get(entityExportUUID);
+ assertNotNull( exportEntity );
+ assertEquals( "COMPLETED",exportEntity.getState().name() );
+ }
+
+ //tests that with empty job data, the export still runs.
+ @Test
+ public void testExportEmptyJobData() throws Exception {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+ }
+
+ @Test
+ public void testNullJobExecution () {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+ }
+
+ @Ignore //For this test please input your s3 credentials into payload builder.
+ public void testIntegration100Entities() throws Exception {
+
+ S3Export s3Export = new S3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[100];
+ //creates entities
+ for (int i = 0; i< 100;i++) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+
+ entity[i] = em.create( "user", userProperties );
+
+ }
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ }
+
+
+ /*Creates fake payload for testing purposes.*/
+ public HashMap<String,Object> payloadBuilder() {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ // TODO: always put dummy values here and ignore this test.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app");
+ payload.put( "properties", properties);
+ return payload;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
new file mode 100644
index 0000000..432fe5e
--- /dev/null
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -0,0 +1,49 @@
+package org.apache.usergrid.management.cassandra;
+
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.management.export.S3Export;
+
+
+/**
+ * Writes to file instead of s3.
+ *
+ */
+public class MockS3ExportImpl implements S3Export {
+ @Override
+ public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
+ Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
+ int read = 0;
+ byte[] bytes = new byte[1024];
+ OutputStream outputStream = null;
+ //FileInputStream fis = new PrintWriter( inputStream );
+
+ try {
+ outputStream = new FileOutputStream( new File("test.json") );
+
+ }
+ catch ( FileNotFoundException e ) {
+ e.printStackTrace();
+ }
+
+
+ try {
+ while ( (read = (inputStream.read( bytes ))) != -1) {
+ outputStream.write( bytes, 0, read );
+ }
+
+ }
+ catch ( IOException e ) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/ServiceITSetup.java b/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
deleted file mode 100644
index 7cd370a..0000000
--- a/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.usergrid;
-
-
-import java.util.Properties;
-
-import org.usergrid.management.ApplicationCreator;
-import org.usergrid.management.ManagementService;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.security.providers.SignInProviderFactory;
-import org.usergrid.security.tokens.TokenService;
-import org.usergrid.services.ServiceManagerFactory;
-
-
-public interface ServiceITSetup extends CoreITSetup {
- ManagementService getMgmtSvc();
-
- ApplicationCreator getAppCreator();
-
- ServiceManagerFactory getSmf();
-
- TokenService getTokenSvc();
-
- Properties getProps();
-
- ExportService getExportService();
-
- /**
- * Convenience method to set a property in the Properties object returned by getProps();
- *
- * @param key the property key
- * @param value the value of the property to set
- *
- * @return the previous value of the property
- */
- Object set( String key, String value );
-
- /**
- * Convenience method to get a property in the Properties object returned by getProps().
- *
- * @param key the property key
- *
- * @return value the value of the property
- */
- String get( String key );
-
- SignInProviderFactory getProviderFactory();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
deleted file mode 100644
index 026b61d..0000000
--- a/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
+++ /dev/null
@@ -1,135 +0,0 @@
-package org.usergrid;
-
-
-import java.util.Properties;
-
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.config.PropertiesFactoryBean;
-import org.usergrid.cassandra.CassandraResource;
-import org.usergrid.management.ApplicationCreator;
-import org.usergrid.management.ManagementService;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.persistence.cassandra.CassandraService;
-import org.usergrid.security.providers.SignInProviderFactory;
-import org.usergrid.security.tokens.TokenService;
-import org.usergrid.services.ServiceManagerFactory;
-
-
-/** A {@link org.junit.rules.TestRule} that sets up services. */
-public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetup {
- private static final Logger LOG = LoggerFactory.getLogger( ServiceITSetupImpl.class );
-
- private ServiceManagerFactory smf;
- private ManagementService managementService;
- private ApplicationCreator applicationCreator;
- private TokenService tokenService;
- private SignInProviderFactory providerFactory;
- private Properties properties;
- private ExportService exportService;
-
-
- public ServiceITSetupImpl( CassandraResource cassandraResource ) {
- super( cassandraResource );
- }
-
-
- protected void after( Description description ) {
- super.after( description );
- LOG.info( "Test {}: finish with application", description.getDisplayName() );
- }
-
-
- protected void before( Description description ) throws Throwable {
- super.before( description );
- managementService = cassandraResource.getBean( ManagementService.class );
- applicationCreator = cassandraResource.getBean( ApplicationCreator.class );
- tokenService = cassandraResource.getBean( TokenService.class );
- providerFactory = cassandraResource.getBean( SignInProviderFactory.class );
- properties = cassandraResource.getBean( PropertiesFactoryBean.class ).getObject();
- smf = cassandraResource.getBean( ServiceManagerFactory.class );
- exportService = cassandraResource.getBean( ExportService.class );
-
- LOG.info( "Test setup complete..." );
- }
-
-
- @Override
- public Statement apply( final Statement base, final Description description ) {
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
- before( description );
-
- try {
- base.evaluate();
- }
- finally {
- after( description );
- }
- }
- };
- }
-
-
- @Override
- public CassandraService getCassSvc() {
- return cassandraResource.getBean( CassandraService.class );
- }
-
-
- @Override
- public ManagementService getMgmtSvc() {
- return managementService;
- }
-
-
- public ServiceManagerFactory getSmf() {
- if ( smf == null ) {
- smf = cassandraResource.getBean( ServiceManagerFactory.class );
- }
-
- return smf;
- }
-
-
- @Override
- public ApplicationCreator getAppCreator() {
- return applicationCreator;
- }
-
-
- @Override
- public TokenService getTokenSvc() {
- return tokenService;
- }
-
-
- @Override
- public Properties getProps() {
- return properties;
- }
-
- @Override
- public ExportService getExportService() { return exportService; }
-
-
- @Override
- public Object set( String key, String value ) {
- return properties.setProperty( key, value );
- }
-
-
- @Override
- public String get( String key ) {
- return properties.getProperty( key );
- }
-
-
- @Override
- public SignInProviderFactory getProviderFactory() {
- return providerFactory;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
deleted file mode 100644
index 41f6259..0000000
--- a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
+++ /dev/null
@@ -1,1152 +0,0 @@
-package org.usergrid.management.cassandra;
-
-
-import java.io.File;
-import java.io.FileReader;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.usergrid.ServiceITSetup;
-import org.usergrid.ServiceITSetupImpl;
-import org.usergrid.ServiceITSuite;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.cassandra.CassandraResource;
-import org.usergrid.cassandra.ClearShiroSubject;
-import org.usergrid.cassandra.Concurrent;
-import org.usergrid.count.SimpleBatcher;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.management.UserInfo;
-import org.usergrid.management.export.ExportJob;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.management.export.S3Export;
-import org.usergrid.management.export.S3ExportImpl;
-import org.usergrid.persistence.CredentialsInfo;
-import org.usergrid.persistence.Entity;
-import org.usergrid.persistence.EntityManager;
-import org.usergrid.persistence.EntityManagerFactory;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.persistence.entities.JobData;
-import org.usergrid.persistence.entities.User;
-import org.usergrid.security.AuthPrincipalType;
-import org.usergrid.security.crypto.command.Md5HashCommand;
-import org.usergrid.security.crypto.command.Sha1HashCommand;
-import org.usergrid.security.tokens.TokenCategory;
-import org.usergrid.security.tokens.exceptions.InvalidTokenException;
-import org.usergrid.utils.JsonUtils;
-import org.usergrid.utils.UUIDUtils;
-
-import org.apache.usergrid.management.cassandra.MockS3ExportImpl;
-
-import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
-import static org.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-
-/** @author zznate */
-@Concurrent()
-public class ManagementServiceIT {
- private static final Logger LOG = LoggerFactory.getLogger( ManagementServiceIT.class );
-
- private static CassandraResource cassandraResource = ServiceITSuite.cassandraResource;
-
- // app-level data generated only once
- private static UserInfo adminUser;
- private static OrganizationInfo organization;
- private static UUID applicationId;
-
- @Rule
- public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
-
- @ClassRule
- public static final ServiceITSetup setup = new ServiceITSetupImpl( cassandraResource );
-
-
- @BeforeClass
- public static void setup() throws Exception {
- LOG.info( "in setup" );
- adminUser = setup.getMgmtSvc().createAdminUser( "edanuff", "Ed Anuff", "ed@anuff.com", "test", false, false );
- organization = setup.getMgmtSvc().createOrganization( "ed-organization", adminUser, true );
- applicationId = setup.getMgmtSvc().createApplication( organization.getUuid(), "ed-application" ).getId();
- }
-
-
- @Test
- public void testGetTokenForPrincipalAdmin() throws Exception {
- String token = ( ( ManagementServiceImpl ) setup.getMgmtSvc() )
- .getTokenForPrincipal( TokenCategory.ACCESS, null, MANAGEMENT_APPLICATION_ID,
- AuthPrincipalType.ADMIN_USER, adminUser.getUuid(), 0 );
- // ^ same as:
- // managementService.getAccessTokenForAdminUser(user.getUuid());
- assertNotNull( token );
- token = ( ( ManagementServiceImpl ) setup.getMgmtSvc() )
- .getTokenForPrincipal( TokenCategory.ACCESS, null, MANAGEMENT_APPLICATION_ID,
- AuthPrincipalType.APPLICATION_USER, adminUser.getUuid(), 0 );
- // This works because ManagementService#getSecret takes the same code
- // path
- // on an OR for APP._USER as for ADMIN_USER
- // is ok technically as ADMIN_USER is a APP_USER to the admin app, but
- // should still
- // be stricter checking
- assertNotNull( token );
- // managementService.getTokenForPrincipal(appUuid, authPrincipal, pUuid,
- // salt, true);
- }
-
-
- @Test
- public void testGetTokenForPrincipalUser() throws Exception {
- // create a user
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "edanuff" );
- properties.put( "email", "ed@anuff.com" );
-
- Entity user = setup.getEmf().getEntityManager( applicationId ).create( "user", properties );
-
- assertNotNull( user );
- String token = ( ( ManagementServiceImpl ) setup.getMgmtSvc() )
- .getTokenForPrincipal( TokenCategory.ACCESS, null, MANAGEMENT_APPLICATION_ID,
- AuthPrincipalType.APPLICATION_USER, user.getUuid(), 0 );
- assertNotNull( token );
- }
-
-
- @Test
- public void testCountAdminUserAction() throws Exception {
- SimpleBatcher batcher = cassandraResource.getBean( SimpleBatcher.class );
-
- batcher.setBlockingSubmit( true );
- batcher.setBatchSize( 1 );
-
- setup.getMgmtSvc().countAdminUserAction( adminUser, "login" );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- Map<String, Long> counts = em.getApplicationCounters();
- LOG.info( JsonUtils.mapToJsonString( counts ) );
- LOG.info( JsonUtils.mapToJsonString( em.getCounterNames() ) );
- assertNotNull( counts.get( "admin_logins" ) );
- assertEquals( 1, counts.get( "admin_logins" ).intValue() );
- }
-
-
- @Test
- public void deactivateUser() throws Exception {
-
- UUID uuid = UUIDUtils.newTimeUUID();
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "test" + uuid );
- properties.put( "email", String.format( "test%s@anuff.com", uuid ) );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
- Entity entity = em.create( "user", properties );
-
- assertNotNull( entity );
-
- User user = em.get( entity.getUuid(), User.class );
-
- assertFalse( user.activated() );
- assertNull( user.getDeactivated() );
-
- setup.getMgmtSvc().activateAppUser( applicationId, user.getUuid() );
-
- user = em.get( entity.getUuid(), User.class );
-
- assertTrue( user.activated() );
- assertNull( user.getDeactivated() );
-
- // get a couple of tokens. These shouldn't work after we deactive the user
- String token1 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, user.getUuid(), 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, user.getUuid(), 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- long startTime = System.currentTimeMillis();
-
- setup.getMgmtSvc().deactivateUser( applicationId, user.getUuid() );
-
- long endTime = System.currentTimeMillis();
-
- user = em.get( entity.getUuid(), User.class );
-
- assertFalse( user.activated() );
- assertNotNull( user.getDeactivated() );
-
- assertTrue( startTime <= user.getDeactivated() && user.getDeactivated() <= endTime );
-
- boolean invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
-
- invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
- }
-
-
- @Test
- public void disableAdminUser() throws Exception {
-
- UUID uuid = UUIDUtils.newTimeUUID();
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "test" + uuid );
- properties.put( "email", String.format( "test%s@anuff.com", uuid ) );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- Entity entity = em.create( "user", properties );
-
- assertNotNull( entity );
-
- User user = em.get( entity.getUuid(), User.class );
-
- assertFalse( user.activated() );
- assertNull( user.getDeactivated() );
-
- setup.getMgmtSvc().activateAdminUser( user.getUuid() );
-
- user = em.get( entity.getUuid(), User.class );
-
- assertTrue( user.activated() );
- assertNull( user.getDeactivated() );
-
- // get a couple of tokens. These shouldn't work after we deactive the user
- String token1 = setup.getMgmtSvc().getAccessTokenForAdminUser( user.getUuid(), 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAdminUser( user.getUuid(), 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().disableAdminUser( user.getUuid() );
-
- user = em.get( entity.getUuid(), User.class );
-
- assertTrue( user.disabled() );
-
- boolean invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
-
- invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
- }
-
-
- @Test
- public void userTokensRevoke() throws Exception {
- UUID userId = UUIDUtils.newTimeUUID();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokensForAppUser( applicationId, userId );
-
- boolean invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
-
- invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
- }
-
-
- @Test
- public void userTokenRevoke() throws Exception {
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "realbeast" );
- properties.put( "email", "sungju@softwaregeeks.org" );
-
- Entity user = em.create( "user", properties );
- assertNotNull( user );
-
- UUID userId = user.getUuid();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokenForAppUser( token1 );
-
- boolean invalidToken1Excpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken1Excpetion = true;
- }
-
- assertTrue( invalidToken1Excpetion );
-
- boolean invalidToken2Excpetion = true;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken2Excpetion = false;
- }
-
- assertTrue( invalidToken2Excpetion );
- }
-
-
- @Test
- public void adminTokensRevoke() throws Exception {
- UUID userId = UUIDUtils.newTimeUUID();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokensForAdminUser( userId );
-
- boolean invalidTokenException = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenException = true;
- }
-
- assertTrue( invalidTokenException );
-
- invalidTokenException = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenException = true;
- }
-
- assertTrue( invalidTokenException );
- }
-
-
- @Test
- public void adminTokenRevoke() throws Exception {
- UUID userId = adminUser.getUuid();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokenForAdminUser( userId, token1 );
-
- boolean invalidToken1Excpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken1Excpetion = true;
- }
-
- assertTrue( invalidToken1Excpetion );
-
- boolean invalidToken2Excpetion = true;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken2Excpetion = false;
- }
-
- assertTrue( invalidToken2Excpetion );
- }
-
-
- @Test
- public void superUserGetOrganizationsPage() throws Exception {
- int beforeSize = setup.getMgmtSvc().getOrganizations().size() - 1;
- // create 15 orgs
- for ( int x = 0; x < 15; x++ ) {
- setup.getMgmtSvc().createOrganization( "super-user-org-" + x, adminUser, true );
- }
- // should be 17 total
- assertEquals( 16 + beforeSize, setup.getMgmtSvc().getOrganizations().size() );
- List<OrganizationInfo> orgs = setup.getMgmtSvc().getOrganizations( null, 10 );
- assertEquals( 10, orgs.size() );
- UUID val = orgs.get( 9 ).getUuid();
- orgs = setup.getMgmtSvc().getOrganizations( val, 10 );
- assertEquals( 7 + beforeSize, orgs.size() );
- assertEquals( val, orgs.get( 0 ).getUuid() );
- }
-
-
- @Test
- public void authenticateAdmin() throws Exception {
-
- String username = "tnine";
- String password = "test";
-
- UserInfo adminUser = setup.getMgmtSvc()
- .createAdminUser( username, "Todd Nine", UUID.randomUUID() + "@apigee.com", password,
- false, false );
-
- UserInfo authedUser = setup.getMgmtSvc().verifyAdminUserPasswordCredentials( username, password );
-
- assertEquals( adminUser.getUuid(), authedUser.getUuid() );
-
- authedUser = setup.getMgmtSvc().verifyAdminUserPasswordCredentials( adminUser.getEmail(), password );
-
- assertEquals( adminUser.getUuid(), authedUser.getUuid() );
-
- authedUser = setup.getMgmtSvc().verifyAdminUserPasswordCredentials( adminUser.getUuid().toString(), password );
-
- assertEquals( adminUser.getUuid(), authedUser.getUuid() );
- }
-
-
- /** Test we can change the password if it's hashed with sha1 */
- @Test
- public void testAdminPasswordChangeShaType() throws Exception {
- String username = "testAdminPasswordChangeShaType";
- String password = "test";
-
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Sha1HashCommand command = new Sha1HashCommand();
- byte[] hashed = command.hash( password.getBytes( "UTF-8" ), info, userId, MANAGEMENT_APPLICATION_ID );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- info.setCipher( command.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( MANAGEMENT_APPLICATION_ID, userId, password, newPassword );
-
- //verify authorization works
- authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
-
- /** Test we can change the password if it's hashed with md5 then sha1 */
- @Test
- public void testAdminPasswordChangeMd5ShaType() throws Exception {
- String username = "testAdminPasswordChangeMd5ShaType";
- String password = "test";
-
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Md5HashCommand md5 = new Md5HashCommand();
-
- Sha1HashCommand sha1 = new Sha1HashCommand();
-
- byte[] hashed = md5.hash( password.getBytes( "UTF-8" ), info, userId, MANAGEMENT_APPLICATION_ID );
- hashed = sha1.hash( hashed, info, userId, MANAGEMENT_APPLICATION_ID );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- //set the final cipher to sha1
- info.setCipher( sha1.getName() );
- //set the next hash type to md5
- info.setHashType( md5.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( MANAGEMENT_APPLICATION_ID, userId, password, newPassword );
-
- //verify authorization works
- authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
-
- @Test
- public void authenticateUser() throws Exception {
-
- String username = "tnine";
- String password = "test";
- String orgName = "autneticateUser";
- String appName = "authenticateUser";
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password );
-
- //verify authorization works
- User authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password, newPassword );
-
- //verify authorization works
- authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, newPassword );
- }
-
-
- /** Test we can change the password if it's hashed with sha1 */
- @Test
- public void testAppUserPasswordChangeShaType() throws Exception {
- String username = "tnine";
- String password = "test";
- String orgName = "testAppUserPasswordChangeShaType";
- String appName = "testAppUserPasswordChangeShaType";
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Sha1HashCommand command = new Sha1HashCommand();
- byte[] hashed = command.hash( password.getBytes( "UTF-8" ), info, userId, appId );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- info.setCipher( command.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password, newPassword );
-
- //verify authorization works
- authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
-
- /** Test we can change the password if it's hashed with md5 then sha1 */
- @Test
- public void testAppUserPasswordChangeMd5ShaType() throws Exception {
- String username = "tnine";
- String password = "test";
- String orgName = "testAppUserPasswordChangeMd5ShaType";
- String appName = "testAppUserPasswordChangeMd5ShaType";
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Md5HashCommand md5 = new Md5HashCommand();
-
- Sha1HashCommand sha1 = new Sha1HashCommand();
-
- byte[] hashed = md5.hash( password.getBytes( "UTF-8" ), info, userId, appId );
- hashed = sha1.hash( hashed, info, userId, appId );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- //set the final cipher to sha1
- info.setCipher( sha1.getName() );
- //set the next hash type to md5
- info.setHashType( md5.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password, newPassword );
-
- //verify authorization works
- authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
- //Tests to make sure we can call the job with mock data and it runs.
- @Ignore
- public void testFileConnections() throws Exception {
-
- File f = null;
-
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
- }
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
- //intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[10];
- //creates entities
- for (int i = 0; i< 10;i++) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
-
- entity[i] = em.create( "user", userProperties );
-
- }
- //creates connections
- em.createConnection( em.getRef( entity[0].getUuid() ),"Vibrations",em.getRef( entity[1].getUuid() ) );
- em.createConnection( em.getRef( entity[1].getUuid() ),"Vibrations",em.getRef( entity[0].getUuid() ) );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- //create and initialize jobData returned in JobExecution.
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
- //assertEquals(13, a.size() );
-
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 0 );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject) objEnt.get( "connections" );
-
- assertNotNull( objConnections );
-
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get("Vibrations");
-
- assertNotNull( objVibrations );
-
- f.delete();
- }
-
-
- @Test
- public void testFileValidity() throws Exception {
-
- File f = null;
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
- }
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
-
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- assertNotNull( entityData );
-
- }
- f.delete();
- }
-
- @Test
- public void testFileExportOneOrg() throws Exception {
-
- File f = null;
-
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
- }
- setup.getMgmtSvc().createOwnerAndOrganization( "noExport","junkUserName","junkRealName","ugExport@usergrid.com","123456789" );
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
-
- //assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
- // assertNotEquals( "NotEqual","junkRealName",entityName );
- assertFalse( "junkRealName".equals( entityName ) );
-
- }
- f.delete();
- }
- @Test
- public void testFileExportOneApp() throws Exception {
-
- File f = null;
- String orgName = "ed-organization";
- String appName = "testAppNotExported";
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
- }
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
- //intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[10];
- //creates entities
- for (int i = 0; i< 10;i++) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
- entity[i] = em.create( "user", userProperties );
- }
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
-
- //assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
- assertFalse( "junkRealName".equals( entityName ) );
- //assertNotEquals( "NotEquals","junkRealName",entityName );
- }
- f.delete();
- }
-
- //only handles the DoJob Code , different tests for DoExport
- @Test
- public void testExportDoJob() throws Exception {
-
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo ); //this needs to be populated with properties of export info
-
- JobExecution jobExecution = mock ( JobExecution.class);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- ExportJob job = new ExportJob();
- ExportService eS = mock (ExportService.class);
- job.setExportService( eS );
- try {
- job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
- }
- assert(true);
-
- }
-
- @Test
- public void testExportDoExport() throws Exception {
-
- EntityManagerFactory emf = setup.getEmf();
- EntityManager em = emf.getEntityManager( applicationId );
- HashMap<String, Object> payload = payloadBuilder();
- ExportService eS = setup.getExportService();
-
- JobExecution jobExecution = mock( JobExecution.class);
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID entityExportUUID = eS.schedule( exportInfo );
-
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", entityExportUUID);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- //Exportem.get(entityExport);
- Export exportEntity = ( Export ) em.get(entityExportUUID);
- assertNotNull( exportEntity );
- String derp = exportEntity.getState().name();
- assertEquals( "PENDING",exportEntity.getState().name());
- try {
- eS.doExport( exportInfo,jobExecution );
- }catch(Exception e) {
- throw e;
- //assert(false);
- }
- exportEntity = ( Export ) em.get(entityExportUUID);
- assertNotNull( exportEntity );
- assertEquals( "COMPLETED",exportEntity.getState().name() );
- }
-
- //tests that with empty job data, the export still runs.
- @Test
- public void testExportEmptyJobData() throws Exception {
-
- JobData jobData = new JobData();
-
- JobExecution jobExecution = mock ( JobExecution.class);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- ExportJob job = new ExportJob();
- S3Export s3Export = mock( S3Export.class );
- setup.getExportService().setS3Export( s3Export );
- job.setExportService( setup.getExportService() );
- try {
- job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
- }
- assert(true);
- }
-
- @Test
- public void testNullJobExecution () {
-
- JobData jobData = new JobData();
-
- JobExecution jobExecution = mock ( JobExecution.class);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- ExportJob job = new ExportJob();
- S3Export s3Export = mock( S3Export.class );
- setup.getExportService().setS3Export( s3Export );
- job.setExportService( setup.getExportService() );
- try {
- job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
- }
- assert(true);
- }
-
- @Ignore //For this test please input your s3 credentials into payload builder.
- public void testIntegration100Entities() throws Exception {
-
- S3Export s3Export = new S3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
- //intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[100];
- //creates entities
- for (int i = 0; i< 100;i++) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
-
- entity[i] = em.create( "user", userProperties );
-
- }
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- //create and initialize jobData returned in JobExecution.
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- }
-
-
- /*Creates fake payload for testing purposes.*/
- public HashMap<String,Object> payloadBuilder() {
- HashMap<String, Object> payload = new HashMap<String, Object>();
- Map<String, Object> properties = new HashMap<String, Object>();
- Map<String, Object> storage_info = new HashMap<String, Object>();
-// TODO: always put dummy values here and ignore this test.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
-
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
-
- payload.put( "path", "test-organization/test-app");
- payload.put( "properties", properties);
- return payload;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java
deleted file mode 100644
index f66b67d..0000000
--- a/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.usergrid.management.cassandra;
-
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.export.S3Export;
-
-
-/**
- * Writes to file instead of s3.
- *
- */
-public class MockS3ExportImpl implements S3Export {
- @Override
- public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
- Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
- int read = 0;
- byte[] bytes = new byte[1024];
- OutputStream outputStream = null;
- //FileInputStream fis = new PrintWriter( inputStream );
-
- try {
- outputStream = new FileOutputStream( new File("test.json") );
-
- }
- catch ( FileNotFoundException e ) {
- e.printStackTrace();
- }
-
-
- try {
- while ( (read = (inputStream.read( bytes ))) != -1) {
- outputStream.write( bytes, 0, read );
- }
-
- }
- catch ( IOException e ) {
- e.printStackTrace();
- }
- }
-}
[6/9] git commit: Removed JobInfo as it wasn’t used at all.
Posted by sn...@apache.org.
Removed JobInfo as it wasn’t used at all.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b1deced8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b1deced8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b1deced8
Branch: refs/pull/60/head
Commit: b1deced83c63b7a200732cdde66a9a7fcd7c1b3d
Parents: e1ab9df
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 09:39:13 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 09:39:13 2014 -0800
----------------------------------------------------------------------
.../java/org/usergrid/management/JobInfo.java | 29 --------------------
1 file changed, 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1deced8/stack/services/src/main/java/org/usergrid/management/JobInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/JobInfo.java b/stack/services/src/main/java/org/usergrid/management/JobInfo.java
deleted file mode 100644
index cc60b0e..0000000
--- a/stack/services/src/main/java/org/usergrid/management/JobInfo.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.usergrid.management;
-
-import java.util.UUID;
-
-/**
- * Created by ApigeeCorporation on 1/31/14.
- */
-//TODO: Documentation on this class
-public class JobInfo {
- private UUID jobId;
-
- public enum StatusType{
- PENDING,STARTED,FAILED,COMPLETED,FAIL1, FAIL2
- }
- private StatusType jobStatusType;
-
- public JobInfo (UUID jobId, StatusType jobStatusType){
- jobId = this.jobId;
- jobStatusType = this.jobStatusType;
- }
-
- public StatusType getStatusType () {
- return jobStatusType;
- }
-
- public UUID getJobId () {
- return jobId;
- }
-}
[2/9] git commit: Added classes to the branch that were labeled
untracked.
Posted by sn...@apache.org.
Added classes to the branch that were labeled untracked.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e1ab9df8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e1ab9df8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e1ab9df8
Branch: refs/pull/60/head
Commit: e1ab9df84e526364ed199dea72988f905add8393
Parents: 22a17dc
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 09:37:04 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 09:37:04 2014 -0800
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/Query.java | 5 +-
.../usergrid/persistence/entities/Export.java | 125 ++
.../usergrid/persistence/entities/Export.java | 124 --
.../rest/management/ManagementResource.java | 15 +-
.../applications/ApplicationResource.java | 84 +-
.../applications/ApplicationResource.java | 277 -----
.../rest/management/ManagementResourceIT.java | 180 ++-
.../apache/usergrid/management/ExportInfo.java | 65 +
.../usergrid/management/export/ExportJob.java | 65 +
.../management/export/ExportService.java | 37 +
.../management/export/ExportServiceImpl.java | 457 +++++++
.../usergrid/management/export/S3Export.java | 16 +
.../management/export/S3ExportImpl.java | 84 ++
.../org/usergrid/management/ExportInfo.java | 65 -
.../usergrid/management/export/ExportJob.java | 64 -
.../management/export/ExportService.java | 37 -
.../management/export/ExportServiceImpl.java | 454 -------
.../usergrid/management/export/S3Export.java | 16 -
.../management/export/S3ExportImpl.java | 84 --
.../resources/usergrid-services-context.xml | 4 +-
.../org/apache/usergrid/ServiceITSetup.java | 3 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 6 +
.../cassandra/ManagementServiceIT.java | 422 +++++++
.../management/cassandra/MockS3ExportImpl.java | 49 +
.../test/java/org/usergrid/ServiceITSetup.java | 47 -
.../java/org/usergrid/ServiceITSetupImpl.java | 135 --
.../cassandra/ManagementServiceIT.java | 1152 ------------------
.../management/cassandra/MockS3ExportImpl.java | 49 -
28 files changed, 1601 insertions(+), 2520 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 01ac231..286cd7f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -36,6 +36,9 @@ import org.antlr.runtime.TokenRewriteStream;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.persistence.Results.Level;
import org.apache.usergrid.persistence.exceptions.QueryParseException;
import org.apache.usergrid.persistence.query.tree.AndOperand;
@@ -51,8 +54,6 @@ import org.apache.usergrid.persistence.query.tree.QueryFilterLexer;
import org.apache.usergrid.persistence.query.tree.QueryFilterParser;
import org.apache.usergrid.utils.JsonUtils;
-import org.apache.commons.lang.StringUtils;
-
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.split;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
new file mode 100644
index 0000000..182b2ae
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
@@ -0,0 +1,125 @@
+package org.apache.usergrid.persistence.entities;
+
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import org.apache.usergrid.persistence.TypedEntity;
+import org.apache.usergrid.persistence.annotations.EntityProperty;
+
+
+/**
+ *Contains state information for an Entity Job
+ *
+ */
+@XmlRootElement
+public class Export extends TypedEntity {
+
+ public static enum State {
+ //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
+ PENDING,STARTED,FAILED,COMPLETED
+ }
+
+ @EntityProperty
+ protected State curState;
+
+ @EntityProperty
+ protected Long queued;
+
+ /** Time send started */
+ @EntityProperty
+ protected Long started;
+
+ /** Time processed */
+ @EntityProperty
+ protected Long finished;
+
+
+ /** Time to expire the exportJob */
+ @EntityProperty
+ protected Long expire;
+
+ /** True if exportJob is canceled */
+ @EntityProperty
+ protected Boolean canceled;
+
+ /** Error message */
+ @EntityProperty
+ protected String errorMessage;
+
+ public Export() {
+ }
+
+ public boolean isExpired () {
+ return (expire != null && expire > System.currentTimeMillis());
+ }
+
+ public Long getStarted() {
+ return started;
+ }
+
+
+ public void setStarted( final Long started ) {
+ this.started = started;
+ }
+
+
+ public Long getFinished() {
+ return finished;
+ }
+
+
+ public void setFinished( final Long finished ) {
+ this.finished = finished;
+ }
+
+
+ public Long getExpire() {
+ return expire;
+ }
+
+
+ public void setExpire( final Long expire ) {
+ this.expire = expire;
+ }
+
+
+ public Boolean getCanceled() {
+ return canceled;
+ }
+
+ //state should moved to a derived state, but it is not there yet.
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public void setState(State setter) {
+ curState = setter;
+ }
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public State getState() { return curState; }
+
+ public void setCanceled( final Boolean canceled ) {
+ this.canceled = canceled;
+ }
+
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+
+ public void setErrorMessage( final String errorMessage ) {
+ this.errorMessage = errorMessage;
+ }
+
+
+ public Long getQueued() {
+ return queued;
+ }
+
+
+ public void setQueued( final Long queued ) {
+ this.queued = queued;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
deleted file mode 100644
index 60d2bcb..0000000
--- a/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
+++ /dev/null
@@ -1,124 +0,0 @@
-package org.usergrid.persistence.entities;
-
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.usergrid.persistence.TypedEntity;
-import org.usergrid.persistence.annotations.EntityProperty;
-
-
-/**
- *Contains state information for an Entity Job
- *
- */
-@XmlRootElement
-public class Export extends TypedEntity {
-
- public static enum State {
- //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
- PENDING,STARTED,FAILED,COMPLETED
- }
-
- @EntityProperty
- protected State curState;
-
- @EntityProperty
- protected Long queued;
-
- /** Time send started */
- @EntityProperty
- protected Long started;
-
- /** Time processed */
- @EntityProperty
- protected Long finished;
-
-
- /** Time to expire the exportJob */
- @EntityProperty
- protected Long expire;
-
- /** True if exportJob is canceled */
- @EntityProperty
- protected Boolean canceled;
-
- /** Error message */
- @EntityProperty
- protected String errorMessage;
-
- public Export() {
- }
-
- public boolean isExpired () {
- return (expire != null && expire > System.currentTimeMillis());
- }
-
- public Long getStarted() {
- return started;
- }
-
-
- public void setStarted( final Long started ) {
- this.started = started;
- }
-
-
- public Long getFinished() {
- return finished;
- }
-
-
- public void setFinished( final Long finished ) {
- this.finished = finished;
- }
-
-
- public Long getExpire() {
- return expire;
- }
-
-
- public void setExpire( final Long expire ) {
- this.expire = expire;
- }
-
-
- public Boolean getCanceled() {
- return canceled;
- }
-
- //state should moved to a derived state, but it is not there yet.
- @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- @EntityProperty
- public void setState(State setter) {
- curState = setter;
- }
- @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- @EntityProperty
- public State getState() { return curState; }
-
- public void setCanceled( final Boolean canceled ) {
- this.canceled = canceled;
- }
-
-
- public String getErrorMessage() {
- return errorMessage;
- }
-
-
- public void setErrorMessage( final String errorMessage ) {
- this.errorMessage = errorMessage;
- }
-
-
- public Long getQueued() {
- return queued;
- }
-
-
- public void setQueued( final Long queued ) {
- this.queued = queued;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 59ceb4e..72e967f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -37,6 +37,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.error.OAuthError;
+import org.apache.amber.oauth2.common.exception.OAuthProblemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.amber.oauth2.common.message.types.GrantType;
+import org.apache.commons.lang.StringUtils;
+import org.apache.shiro.codec.Base64;
+
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.management.exceptions.DisabledAdminUserException;
import org.apache.usergrid.management.exceptions.UnactivatedAdminUserException;
@@ -48,13 +56,6 @@ import org.apache.usergrid.rest.management.users.UsersResource;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.shiro.utils.SubjectUtils;
-import org.apache.amber.oauth2.common.error.OAuthError;
-import org.apache.amber.oauth2.common.exception.OAuthProblemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.amber.oauth2.common.message.types.GrantType;
-import org.apache.commons.lang.StringUtils;
-import org.apache.shiro.codec.Base64;
-
import com.sun.jersey.api.view.Viewable;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index dff0924..5f45326 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -16,6 +16,7 @@
package org.apache.usergrid.rest.management.organizations.applications;
+import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@@ -25,30 +26,44 @@ import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthSystemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.ExportInfo;
import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+import org.apache.usergrid.rest.utils.JSONPUtils;
import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
import org.apache.usergrid.security.providers.SignInAsProvider;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.services.ServiceManager;
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
+import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
+import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
@@ -60,6 +75,8 @@ import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
})
public class ApplicationResource extends AbstractContextResource {
+ @Autowired
+ protected ExportService exportService;
OrganizationInfo organization;
UUID applicationId;
ApplicationInfo application;
@@ -192,4 +209,67 @@ public class ApplicationResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+
+ @POST
+ @Path("export")
+ @Consumes(APPLICATION_JSON)
+ @RequireOrganizationAccess
+ public Response exportPostJson (@Context UriInfo ui,
+ Map<String, Object> json,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
+ throws OAuthSystemException {
+
+
+ OAuthResponse response = null;
+ UUID jobUUID = null;
+ Map<String, String> uuidRet = new HashMap<String, String>( );
+
+ try {
+ //parse the json into some useful object (the config params)
+ ExportInfo objEx = new ExportInfo(json);
+ objEx.setApplicationId( applicationId );
+ jobUUID = exportService.schedule(objEx);
+ uuidRet.put( "jobUUID", jobUUID.toString() );
+
+ }
+ catch (NullPointerException e) {
+ OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
+ .setErrorDescription( "Job Not Created" )
+ .buildJSONMessage();
+
+ return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
+ .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
+ }
+ catch (Exception e) {
+ //TODO:throw descriptive error message and or include on in the response
+ //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
+ return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
+ }
+
+ return Response.status(SC_ACCEPTED).entity(uuidRet).build();
+
+ //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
+ // .entity( wrapWithCallback( "", callback ) ).build();
+ }
+
+ @GET
+ @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
+ public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
+
+ Export entity;
+ try {
+ entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
+ }catch(Exception e) {
+ return Response.status( SC_BAD_REQUEST ).build();
+ }
+ //validate this user owns it
+
+ if (entity == null) {
+ return Response.status(SC_BAD_REQUEST).build();
+ }
+
+ return Response.status(SC_OK).entity(entity.getState()).build();
+ //return Response.status(SC_OK).entity(state).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
deleted file mode 100644
index 5a29500..0000000
--- a/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*******************************************************************************
- * Copyright 2012 Apigee Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.usergrid.rest.management.organizations.applications;
-
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-import org.usergrid.management.ApplicationInfo;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.rest.AbstractContextResource;
-import org.usergrid.rest.ApiResponse;
-import org.usergrid.rest.applications.ServiceResource;
-import org.usergrid.rest.security.annotations.RequireOrganizationAccess;
-import org.usergrid.rest.utils.JSONPUtils;
-import org.usergrid.security.oauth.ClientCredentialsInfo;
-import org.usergrid.security.providers.SignInAsProvider;
-import org.usergrid.security.providers.SignInProviderFactory;
-import org.usergrid.services.ServiceManager;
-
-import org.apache.amber.oauth2.common.exception.OAuthSystemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.commons.lang.StringUtils;
-
-import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-
-import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
-import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
-import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
-
-
-@Component("org.usergrid.rest.management.organizations.applications.ApplicationResource")
-@Scope("prototype")
-@Produces({
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-})
-public class ApplicationResource extends AbstractContextResource {
-
- @Autowired
- protected ExportService exportService;
- OrganizationInfo organization;
- UUID applicationId;
- ApplicationInfo application;
-
- @Autowired
- private SignInProviderFactory signInProviderFactory;
-
-
- public ApplicationResource() {
- }
-
-
- public ApplicationResource init( OrganizationInfo organization, UUID applicationId ) {
- this.organization = organization;
- this.applicationId = applicationId;
- return this;
- }
-
-
- public ApplicationResource init( OrganizationInfo organization, ApplicationInfo application ) {
- this.organization = organization;
- applicationId = application.getId();
- this.application = application;
- return this;
- }
-
-
- @RequireOrganizationAccess
- @DELETE
- public JSONWithPadding deleteApplicationFromOrganizationByApplicationId( @Context UriInfo ui,
- @QueryParam("callback")
- @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "delete application from organization" );
-
- management.deleteOrganizationApplication( organization.getUuid(), applicationId );
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @GET
- public JSONWithPadding getApplication( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- ServiceManager sm = smf.getServiceManager( applicationId );
- response.setAction( "get" );
- response.setApplication( sm.getApplication() );
- response.setParams( ui.getQueryParameters() );
- response.setResults( management.getApplicationMetadata( applicationId ) );
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @GET
- @Path("credentials")
- public JSONWithPadding getCredentials( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "get application client credentials" );
-
- ClientCredentialsInfo credentials =
- new ClientCredentialsInfo( management.getClientIdForApplication( applicationId ),
- management.getClientSecretForApplication( applicationId ) );
-
- response.setCredentials( credentials );
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @POST
- @Path("credentials")
- public JSONWithPadding generateCredentials( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "generate application client credentials" );
-
- ClientCredentialsInfo credentials =
- new ClientCredentialsInfo( management.getClientIdForApplication( applicationId ),
- management.newClientSecretForApplication( applicationId ) );
-
- response.setCredentials( credentials );
- return new JSONWithPadding( response, callback );
- }
-
-
- @POST
- @Path("sia-provider")
- @Consumes(APPLICATION_JSON)
- @RequireOrganizationAccess
- public JSONWithPadding configureProvider( @Context UriInfo ui, @QueryParam("provider_key") String siaProvider,
- Map<String, Object> json,
- @QueryParam("callback") @DefaultValue("") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "post signin provider configuration" );
-
- Preconditions.checkArgument( siaProvider != null, "Sign in provider required" );
-
- SignInAsProvider signInAsProvider = null;
- if ( StringUtils.equalsIgnoreCase( siaProvider, "facebook" ) ) {
- signInAsProvider =
- signInProviderFactory.facebook( smf.getServiceManager( applicationId ).getApplication() );
- }
- else if ( StringUtils.equalsIgnoreCase( siaProvider, "pingident" ) ) {
- signInAsProvider =
- signInProviderFactory.pingident( smf.getServiceManager( applicationId ).getApplication() );
- }
- else if ( StringUtils.equalsIgnoreCase( siaProvider, "foursquare" ) ) {
- signInAsProvider =
- signInProviderFactory.foursquare( smf.getServiceManager( applicationId ).getApplication() );
- }
-
- Preconditions
- .checkArgument( signInAsProvider != null, "No signin provider found by that name: " + siaProvider );
-
- signInAsProvider.saveToConfiguration( json );
-
- return new JSONWithPadding( response, callback );
- }
- //add export here
- @POST
- @Path("export")
- @Consumes(APPLICATION_JSON)
- @RequireOrganizationAccess
- public Response exportPostJson (@Context UriInfo ui,
- Map<String, Object> json,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
- throws OAuthSystemException {
-
-
- OAuthResponse response = null;
- UUID jobUUID = null;
- Map<String, String> uuidRet = new HashMap<String, String>( );
-
- try {
- //parse the json into some useful object (the config params)
- ExportInfo objEx = new ExportInfo(json);
- objEx.setApplicationId( applicationId );
- jobUUID = exportService.schedule(objEx);
- uuidRet.put( "jobUUID", jobUUID.toString() );
-
- }
- catch (NullPointerException e) {
- OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
- .setErrorDescription( "Job Not Created" )
- .buildJSONMessage();
-
- return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
- .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
- }
- catch (Exception e) {
- //TODO:throw descriptive error message and or include on in the response
- //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
- return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
- }
-
- return Response.status(SC_ACCEPTED).entity(uuidRet).build();
-
- //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
- // .entity( wrapWithCallback( "", callback ) ).build();
- }
-
- @GET
- @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
- public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
-
- Export entity;
- try {
- entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
- }catch(Exception e) {
- return Response.status( SC_BAD_REQUEST ).build();
- }
- //validate this user owns it
-
- if (entity == null) {
- return Response.status(SC_BAD_REQUEST).build();
- }
-
- return Response.status(SC_OK).entity(entity.getState()).build();
- //return Response.status(SC_OK).entity(state).build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index b379770..0621219 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -18,28 +18,31 @@ package org.apache.usergrid.rest.management;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
import javax.ws.rs.core.MediaType;
import org.codehaus.jackson.JsonNode;
+import org.junit.Ignore;
import org.junit.Test;
+
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.cassandra.Concurrent;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.OrganizationOwnerInfo;
import org.apache.usergrid.rest.AbstractRestIT;
import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
-import org.apache.commons.lang.StringUtils;
-
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.representation.Form;
+import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.apache.usergrid.utils.MapUtils.hashMap;
/** @author tnine */
@@ -528,4 +531,175 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, status );
}
+ @Test
+ public void exportCallSuccessful() throws Exception {
+ Status responseStatus = Status.OK;
+ JsonNode node = null;
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.OK, responseStatus );
+ }
+
+ @Ignore
+ public void exportCallCreationEntities100() throws Exception {
+ Status responseStatus = Status.OK;
+ JsonNode node = null;
+
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ //TODO: make sure to put a valid admin token here.
+ storage_info.put( "admin_token", adminToken() );
+ //TODO: always put dummy values here and ignore this test.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app/user");
+ payload.put( "properties", properties);
+
+ for (int i = 0; i < 100; i++) {
+ Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred"+i );
+
+ node = resource().path( "/test-organization/test-app/app_users" )
+ .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, userCreation );
+ }
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.OK, responseStatus );
+ }
+
+ @Test
+ public void exportUUIDRetTest() throws Exception {
+ Status responseStatus = Status.ACCEPTED;
+ String uuid;
+ UUID jobUUID = null;
+ JsonNode node = null;
+
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.ACCEPTED, responseStatus );
+ assertNotNull( node.get( "jobUUID" ) );
+ }
+ /*Make a test with an invalid uuid and a wrong uuid.*/
+ //all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
+ @Test
+ public void exportGetJobStatTest() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ String uuid = String.valueOf( node.get( "jobUUID" ) );
+ uuid = uuid.replaceAll( "\"","" );
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/"+uuid).queryParam( "access_token",
+ superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ // node = resource().path( "/management/exportStats/"+uuid ).accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ // .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+
+ assertEquals( Status.OK, responseStatus );
+ assertEquals( "PENDING",node.asText() );//TODO: do tests for other states in service tier
+
+ }
+ //do an unauthorized test for both post and get
+ @Test
+ public void exportGetWrongUUID() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+ try {
+ node = resource().path(
+ "/management/orgs/test-organization/apps/test-app/export/"+fake ).queryParam( "access_token",
+ superAdminToken() ).accept( MediaType.APPLICATION_JSON ).type(
+ MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+ assertEquals(Status.BAD_REQUEST, responseStatus);
+
+ }
+
+ @Test
+ public void exportGetUnauthorized() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+ assertEquals(Status.UNAUTHORIZED, responseStatus);
+ }
+
+ /*Creates fake payload for testing purposes.*/
+ public HashMap<String,Object> payloadBuilder() {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ //TODO: make sure to put a valid admin token here.
+ //storage_info.put( "admin_token","insert_token_data_here" );
+ //TODO: always put dummy values here and ignore this test.
+ //TODO: add a ret for when s3 values are invalid.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app");
+ payload.put( "properties", properties);
+ return payload;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
new file mode 100644
index 0000000..f4ed4cf
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -0,0 +1,65 @@
+package org.apache.usergrid.management;
+
+
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.TypedEntity;
+
+
+/**
+ * Created by ApigeeCorporation on 1/31/14.
+ */
+//TODO: Documentation on this class.
+public class ExportInfo extends TypedEntity {
+
+ private String path;
+ private Map<String, Object> properties;
+ private String storage_provider;
+ private Map<String, Object> storage_info;
+ private String s3_accessId;
+ private String s3_key;
+ private String bucket_location;
+ private UUID applicationId;
+
+ public ExportInfo ( Map<String, Object> exportData) {
+ path = (String) exportData.get("path");
+ properties = (Map) exportData.get("properties");
+ storage_provider = (String) properties.get ("storage_provider");
+ storage_info = (Map) properties.get("storage_info");
+ s3_accessId = (String) storage_info.get("s3_accessId");
+ s3_key = (String) storage_info.get("s3_key");
+ bucket_location = (String) storage_info.get("bucket_location");
+ }
+
+
+ public UUID getApplicationId() {
+ return applicationId;
+ }
+
+
+ public String getPath () {
+ return path;
+ }
+
+ //Wouldn't get exposed.
+ public Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ public String getStorage_provider () {
+ return storage_provider;
+ }
+ //TODO: write setter methods
+
+ public Map<String, Object> getStorage_info () { return storage_info; }
+
+ //TODO: is this a security concern? How would we get rid of the key once we're done with this value?
+ public String getS3_key () { return s3_key; }
+
+ public String getBucket_location () { return bucket_location; }
+
+ public String getS3_accessId () { return s3_accessId; }
+
+ public void setApplicationId (UUID appId) { applicationId = appId;}
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
new file mode 100644
index 0000000..9830040
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
@@ -0,0 +1,65 @@
+package org.apache.usergrid.management.export;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.job.OnlyOnceJob;
+import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.persistence.entities.JobData;
+
+
+/**
+ *
+ *you could make an enum here, that contains the state info
+ * look at scotts code and emulate that to see wha tyou can return in the json object
+ */
+@Component("exportJob")
+public class ExportJob extends OnlyOnceJob {
+ private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
+
+ @Autowired
+ ExportService exportService;
+
+ public ExportJob() {
+ logger.info("ExportJob created " + this);
+ }
+
+ @Override
+ public void doJob(JobExecution jobExecution) throws Exception {
+ logger.info( "execute ExportJob {}", jobExecution );
+
+ ExportInfo config = null;
+ //add check for null jobData.
+ JobData jobData = jobExecution.getJobData();
+ config = (ExportInfo) jobData.getProperty( "exportInfo" );
+ if(jobData == null) {
+ logger.error( "jobData cannot be null" );
+ return;
+ }
+ else if (config == null) {
+ logger.error( "Export information cannot be null" );
+ return;
+ }
+ else {
+ jobExecution.heartbeat();
+ exportService.doExport( config,jobExecution );
+ }
+ logger.info( "executed ExportJob process completed" );
+ }
+
+ @Override
+ protected long getDelay(JobExecution jobExecution) throws Exception {
+ //return arbitrary number
+ return 100;
+ }
+
+
+ @Autowired
+ public void setExportService( final ExportService exportService ) {
+ this.exportService = exportService;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
new file mode 100644
index 0000000..6e9f4e5
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -0,0 +1,37 @@
+package org.apache.usergrid.management.export;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.management.ExportInfo;
+
+
+/**
+ * Performs all functions related to exporting
+ *
+ */
+public interface ExportService {
+
+ /**
+ * Schedules the export to execute
+ * @param config
+ */
+ UUID schedule(ExportInfo config) throws Exception;
+
+
+ /**
+ * Perform the export to the external resource
+ * @param config
+ */
+ void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
+
+ /**
+ * Returns the current state of the service.
+ * @return
+ */
+ String getState(UUID appId,UUID state) throws Exception;
+
+ void setS3Export(S3Export s3Export);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
new file mode 100644
index 0000000..2d8ff5c
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -0,0 +1,457 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.service.SchedulerService;
+import org.apache.usergrid.management.ExportInfo;
+
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.PagingResultsIterator;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.entities.Export;
+import org.apache.usergrid.persistence.entities.JobData;
+
+import com.google.common.collect.BiMap;
+
+
+/**
+ *
+ *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
+ * it will just be the one app and the one org and all of it's collections.
+ */
+public class ExportServiceImpl implements ExportService {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+ public static final String EXPORT_ID = "exportId";
+ public static final String EXPORT_JOB_NAME = "exportJob";
+ //dependency injection
+ private SchedulerService sch;
+
+ //injected the Entity Manager Factory
+ protected EntityManagerFactory emf;
+
+ //inject Management Service to access Organization Data
+ private ManagementService managementService;
+
+ //Maximum amount of entities retrieved in a single go.
+ public static final int MAX_ENTITY_FETCH = 100;
+
+ //Amount of time that has passed before sending another heart beat in millis
+ public static final int TIMESTAMP_DELTA = 5000;
+
+ private JsonFactory jsonFactory = new JsonFactory();
+
+ protected long startTime = System.currentTimeMillis();
+
+ private S3Export s3Export;
+
+ @Override
+ public UUID schedule( final ExportInfo config ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( config.getApplicationId());
+
+ Export export = new Export();
+ export.setState( Export.State.PENDING );
+
+ //validate that org exists,then app, then collection.
+
+ String pathToBeParsed = config.getPath();
+ //split the path so that you can verify that the organization and the app exist.
+ String[] pathItems = pathToBeParsed.split( "/" );
+
+
+ try {
+ managementService.getOrganizationByName( pathItems[0] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Organization doesn't exist" );
+ }
+
+ try {
+ managementService.getApplicationInfo( pathItems[1] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Application doesn't exist" );
+ }
+
+
+ //TODO: parse path and make sure all the things you need actually exist. then throw
+ // good error messages when not found.
+
+ //write to the em
+ export = em.create( export );
+ export.setState( Export.State.PENDING );
+ em.update( export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "exportInfo", config );
+ jobData.setProperty( EXPORT_ID, export.getUuid() );
+
+ long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+
+ sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
+
+ return export.getUuid();
+
+ }
+ /**
+ * Query Entity Manager for specific Export Entity within application
+ * @param appId,uuid
+ * @return String
+ * @throws Exception
+ */
+ @Override
+ public String getState(final UUID appId, final UUID uuid) throws Exception {
+
+ EntityManager rootEm = emf.getEntityManager( appId );
+ Export export = rootEm.get( uuid, Export.class );
+
+ if(export == null){
+ return null;
+ }
+ return export.getState().toString();
+ }
+
+
+ @Override
+ public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
+
+ UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+ EntityManager em = emf.getEntityManager( config.getApplicationId());
+ Export export = em.get( exportId, Export.class );
+
+ String pathToBeParsed = config.getPath();
+ String[] pathItems = pathToBeParsed.split( "/" );
+ try {
+ managementService.getOrganizationByName( pathItems[0] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Organization doesn't exist" );
+ return;
+ }
+
+ //update state and re-write the entity
+ export.setState( Export.State.STARTED );
+
+ em.update( export );
+
+ Map<UUID, String> organizationGet = getOrgs(config);
+ for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
+ //needs to pass app name, and possibly collection to export
+ exportApplicationsForOrg( organization, config, jobExecution );
+ }
+ export.setState( Export.State.COMPLETED );
+ em.update( export );
+ }
+
+ private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
+ // Loop through the organizations
+ // TODO:this will come from the orgs in schedule when you do the validations. delete orgId
+ UUID orgId = null;
+
+ Map<UUID, String> organizationNames = null;
+
+ if ( orgId == null ) {
+ organizationNames = managementService.getOrganizations();
+ }
+
+ else {
+ OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
+
+ if ( info == null ) {
+ logger.error( "Organization info is null!" );
+ }
+
+ organizationNames = new HashMap<UUID, String>();
+ organizationNames.put( orgId, info.getName() );
+ }
+
+
+ return organizationNames;
+ }
+
+
+ public SchedulerService getSch() {
+ return sch;
+ }
+
+
+ public void setSch( final SchedulerService sch ) {
+ this.sch = sch;
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ public void setEmf( final EntityManagerFactory emf ) {
+ this.emf = emf;
+ }
+
+
+ public ManagementService getManagementService() {
+
+ return managementService;
+ }
+
+ public void setManagementService( final ManagementService managementService ) {
+ this.managementService = managementService;
+ }
+
+ //write test checking to see what happens if the input stream is closed or wrong.
+ //TODO: make multipart streaming functional
+ //currently only stores the collection in memory then flushes it.
+ private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
+ final JobExecution jobExecution ) throws Exception {
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ logger.info( "" + organization );
+
+ // Loop through the applications per organization
+ BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organization.getKey() );
+ for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+
+ logger.info( application.getValue() + " : " + application.getKey() );
+
+ String appFileName = prepareOutputFileName( "application", application.getValue() );
+
+ JsonGenerator jg = getJsonGenerator( baos );
+
+ EntityManager em = emf.getEntityManager( application.getKey() );
+
+ jg.writeStartArray();
+
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+ long starting_time = System.currentTimeMillis();
+
+ // Loop through the collections. This is the only way to loop
+ // through the entities in the application (former namespace).
+ //could support queries, just need to implement that in the rest endpoint.
+ for ( String collectionName : metadata.keySet() ) {
+ if(collectionName.equals( "exports" )) {
+ continue;
+ }
+
+ Query query = new Query();
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+
+ for( Object e: itr){
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+
+ jg.writeStartObject();
+ jg.writeFieldName( "Metadata" );
+ jg.writeObject( entity );
+ saveCollectionMembers( jg, em, application.getValue(), entity );
+ jg.writeEndObject();
+ }
+ }
+
+ // Close writer and file for this application.
+ jg.writeEndArray();
+ jg.close();
+ baos.flush();
+ baos.close();
+
+ InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+ s3Export.copyToS3( is, config , appFileName);
+ }
+ }
+
+
+ public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) {
+
+ long cur_time = System.currentTimeMillis();
+
+ if ( startingTime <= ( cur_time - TIMESTAMP_DELTA ) ) {
+ jobExecution.heartbeat();
+ return cur_time;
+ }
+ return startingTime;
+ }
+
+
+ /**
+ * Serialize and save the collection members of this <code>entity</code>
+ *
+ * @param em Entity Manager
+ * @param application Application name
+ * @param entity entity
+ */
+ private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity )
+ throws Exception {
+
+ Set<String> collections = em.getCollections( entity );
+ //jg.writeStartObject();
+
+ // Only create entry for Entities that have collections
+ if ( ( collections == null ) || collections.isEmpty() ) {
+ return;
+ }
+
+
+ for ( String collectionName : collections ) {
+
+ jg.writeFieldName( collectionName );
+ jg.writeStartArray();
+
+ //is 100000 an arbitary number?
+ Results collectionMembers =
+ em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
+
+ List<UUID> entityIds = collectionMembers.getIds();
+
+ if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
+ for ( UUID childEntityUUID : entityIds ) {
+ jg.writeObject( childEntityUUID.toString() );
+ }
+ }
+
+ // End collection array.
+ jg.writeEndArray();
+ }
+
+ // Write connections
+ saveConnections( entity, em, jg );
+
+ // Write dictionaries
+ saveDictionaries( entity, em, jg );
+
+ // End the object if it was Started
+ //jg.writeEndObject();
+ }
+
+
+ /**
+ * Persists the connection for this entity.
+ */
+ private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "dictionaries" );
+ jg.writeStartObject();
+
+ Set<String> dictionaries = em.getDictionaries( entity );
+ for ( String dictionary : dictionaries ) {
+
+ Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
+
+ // nothing to do
+ if ( dict.isEmpty() ) {
+ continue;
+ }
+
+ jg.writeFieldName( dictionary );
+
+ jg.writeStartObject();
+
+ for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
+ jg.writeFieldName( entry.getKey().toString() );
+ jg.writeObject( entry.getValue() );
+ }
+
+ jg.writeEndObject();
+ }
+ jg.writeEndObject();
+ }
+
+
+ /**
+ * Persists the connection for this entity.
+ */
+ private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "connections" );
+ jg.writeStartObject();
+
+ Set<String> connectionTypes = em.getConnectionTypes( entity );
+ for ( String connectionType : connectionTypes ) {
+
+ jg.writeFieldName( connectionType );
+ jg.writeStartArray();
+
+ Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Results.Level.IDS );
+ List<ConnectionRef> connections = results.getConnections();
+
+ for ( ConnectionRef connectionRef : connections ) {
+ jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
+ }
+
+ jg.writeEndArray();
+ }
+ jg.writeEndObject();
+ }
+
+
+ protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
+ //TODO:shouldn't the below be UTF-16?
+ //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
+
+ JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+ jg.setPrettyPrinter( new DefaultPrettyPrinter() );
+ jg.setCodec( new ObjectMapper() );
+ return jg;
+ }
+
+
+ protected File createOutputFile( String type, String name ) {
+ return new File( prepareOutputFileName( type, name ) );
+ }
+
+
+ /**
+ * @param type just a label such us: organization, application.
+ *
+ * @return the file name concatenated with the type and the name of the collection
+ */
+ protected String prepareOutputFileName( String type, String name ) {
+ StringBuilder str = new StringBuilder();
+ str.append( name );
+ str.append( "." );
+ str.append( startTime );
+ str.append( ".json" );
+
+ String outputFileName = str.toString();
+ //TODO:this is , i feel, bad practice so make sure to come back here and fix it up.
+
+ return outputFileName;
+ }
+
+
+ @Autowired
+ @Override
+ public void setS3Export( S3Export s3Export ) { this.s3Export = s3Export; }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
new file mode 100644
index 0000000..941a612
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -0,0 +1,16 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.InputStream;
+
+import org.apache.usergrid.management.ExportInfo;
+
+
+/**
+ *
+ *
+ */
+public interface S3Export {
+ void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename );
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
new file mode 100644
index 0000000..54d8ab1
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -0,0 +1,84 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.ExportInfo;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Module;
+
+
+/**
+ *
+ *
+ */
+public class S3ExportImpl implements S3Export {
+
+ @Override
+ public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
+
+ Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+ /*won't need any of the properties as I have the export info*/
+ String bucketName = exportInfo.getBucket_location();
+ String accessId = exportInfo.getS3_accessId();
+ String secretKey = exportInfo.getS3_key();
+
+ Properties overrides = new Properties();
+ overrides.setProperty( "s3" + ".identity", accessId );
+ overrides.setProperty( "s3" + ".credential", secretKey );
+
+ final Iterable<? extends Module> MODULES = ImmutableSet
+ .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule
+ () );
+
+ BlobStoreContext context =
+ ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
+ .overrides( overrides ).buildView( BlobStoreContext.class );
+
+ // Create Container (the bucket in s3)
+ try {
+ AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
+ // BlobStore (returns false if it already exists)
+ ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName );
+ if ( container.get() ) {
+ logger.info( "Created bucket " + bucketName );
+ }
+ }
+ catch ( Exception ex ) {
+ logger.error( "Could not start binary service: {}", ex.getMessage() );
+ return;
+ }
+
+ try {
+ AsyncBlobStore blobStore = context.getAsyncBlobStore();
+ BlobBuilder blobBuilder =
+ blobStore.blobBuilder( filename ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+
+
+ Blob blob = blobBuilder.build();
+
+ ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+
+ logger.info( "Uploaded file etag=" + futureETag.get() );
+ }
+ catch ( Exception e ) {
+ logger.error( "Error uploading to blob store", e );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
deleted file mode 100644
index 7821ca6..0000000
--- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.usergrid.management;
-
-
-import java.util.Map;
-import java.util.UUID;
-
-import org.usergrid.persistence.TypedEntity;
-
-
-/**
- * Created by ApigeeCorporation on 1/31/14.
- */
-//TODO: Documentation on this class.
-public class ExportInfo extends TypedEntity {
-
- private String path;
- private Map<String, Object> properties;
- private String storage_provider;
- private Map<String, Object> storage_info;
- private String s3_accessId;
- private String s3_key;
- private String bucket_location;
- private UUID applicationId;
-
- public ExportInfo ( Map<String, Object> exportData) {
- path = (String) exportData.get("path");
- properties = (Map) exportData.get("properties");
- storage_provider = (String) properties.get ("storage_provider");
- storage_info = (Map) properties.get("storage_info");
- s3_accessId = (String) storage_info.get("s3_accessId");
- s3_key = (String) storage_info.get("s3_key");
- bucket_location = (String) storage_info.get("bucket_location");
- }
-
-
- public UUID getApplicationId() {
- return applicationId;
- }
-
-
- public String getPath () {
- return path;
- }
-
- //Wouldn't get exposed.
- public Map<String, Object> getProperties() {
- return properties;
- }
-
- public String getStorage_provider () {
- return storage_provider;
- }
- //TODO: write setter methods
-
- public Map<String, Object> getStorage_info () { return storage_info; }
-
- //TODO: is this a security concern? How would we get rid of the key once we're done with this value?
- public String getS3_key () { return s3_key; }
-
- public String getBucket_location () { return bucket_location; }
-
- public String getS3_accessId () { return s3_accessId; }
-
- public void setApplicationId (UUID appId) { applicationId = appId;}
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
deleted file mode 100644
index ed8a24e..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.usergrid.management.export;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.batch.job.OnlyOnceJob;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.persistence.entities.JobData;
-
-
-/**
- *
- *you could make an enum here, that contains the state info
- * look at scotts code and emulate that to see wha tyou can return in the json object
- */
-@Component("exportJob")
-public class ExportJob extends OnlyOnceJob {
- private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
-
- @Autowired
- ExportService exportService;
-
- public ExportJob() {
- logger.info("ExportJob created " + this);
- }
-
- @Override
- public void doJob(JobExecution jobExecution) throws Exception {
- logger.info( "execute ExportJob {}", jobExecution );
-
- ExportInfo config = null;
- //add check for null jobData.
- JobData jobData = jobExecution.getJobData();
- config = (ExportInfo) jobData.getProperty( "exportInfo" );
- if(jobData == null) {
- logger.error( "jobData cannot be null" );
- return;
- }
- else if (config == null) {
- logger.error( "Export information cannot be null" );
- return;
- }
- else {
- jobExecution.heartbeat();
- exportService.doExport( config,jobExecution );
- }
- logger.info( "executed ExportJob process completed" );
- }
-
- @Override
- protected long getDelay(JobExecution jobExecution) throws Exception {
- //return arbitrary number
- return 100;
- }
-
-
- @Autowired
- public void setExportService( final ExportService exportService ) {
- this.exportService = exportService;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
deleted file mode 100644
index 644aef6..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.util.UUID;
-
-import org.usergrid.batch.JobExecution;
-import org.usergrid.management.ExportInfo;
-
-
-/**
- * Performs all functions related to exporting
- *
- */
-public interface ExportService {
-
- /**
- * Schedules the export to execute
- * @param config
- */
- UUID schedule(ExportInfo config) throws Exception;
-
-
- /**
- * Perform the export to the external resource
- * @param config
- */
- void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
-
- /**
- * Returns the current state of the service.
- * @return
- */
- String getState(UUID appId,UUID state) throws Exception;
-
- void setS3Export(S3Export s3Export);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
deleted file mode 100644
index 498828b..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ /dev/null
@@ -1,454 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.util.DefaultPrettyPrinter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.batch.service.SchedulerService;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.ManagementService;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.persistence.ConnectionRef;
-import org.usergrid.persistence.Entity;
-import org.usergrid.persistence.EntityManager;
-import org.usergrid.persistence.EntityManagerFactory;
-import org.usergrid.persistence.PagingResultsIterator;
-import org.usergrid.persistence.Query;
-import org.usergrid.persistence.Results;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.persistence.entities.JobData;
-
-import com.google.common.collect.BiMap;
-
-
-/**
- *
- *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
- * it will just be the one app and the one org and all of it's collections.
- */
-public class ExportServiceImpl implements ExportService {
-
-
- private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
- public static final String EXPORT_ID = "exportId";
- public static final String EXPORT_JOB_NAME = "exportJob";
- //dependency injection
- private SchedulerService sch;
-
- //injected the Entity Manager Factory
- protected EntityManagerFactory emf;
-
- //inject Management Service to access Organization Data
- private ManagementService managementService;
-
- //Maximum amount of entities retrieved in a single go.
- public static final int MAX_ENTITY_FETCH = 100;
-
- //Amount of time that has passed before sending another heart beat in millis
- public static final int TIMESTAMP_DELTA = 5000;
-
- private JsonFactory jsonFactory = new JsonFactory();
-
- protected long startTime = System.currentTimeMillis();
-
- private S3Export s3Export;
-
- @Override
- public UUID schedule( final ExportInfo config ) throws Exception {
-
- EntityManager em = emf.getEntityManager( config.getApplicationId());
-
- Export export = new Export();
- export.setState( Export.State.PENDING );
-
- //validate that org exists,then app, then collection.
-
- String pathToBeParsed = config.getPath();
- //split the path so that you can verify that the organization and the app exist.
- String[] pathItems = pathToBeParsed.split( "/" );
-
-
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- }
-
- try {
- managementService.getApplicationInfo( pathItems[1] );
- }
- catch ( Exception e ) {
- logger.error( "Application doesn't exist" );
- }
-
-
- //TODO: parse path and make sure all the things you need actually exist. then throw
- // good error messages when not found.
-
- //write to the em
- export = em.create( export );
- export.setState( Export.State.PENDING );
- em.update( export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "exportInfo", config );
- jobData.setProperty( EXPORT_ID, export.getUuid() );
-
- long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
-
- sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
-
- return export.getUuid();
-
- }
- /**
- * Query Entity Manager for specific Export Entity within application
- * @param appId,uuid
- * @return String
- * @throws Exception
- */
- @Override
- public String getState(final UUID appId, final UUID uuid) throws Exception {
-
- EntityManager rootEm = emf.getEntityManager( appId );
- Export export = rootEm.get( uuid, Export.class );
-
- if(export == null){
- return null;
- }
- return export.getState().toString();
- }
-
-
- @Override
- public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
-
- UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- EntityManager em = emf.getEntityManager( config.getApplicationId());
- Export export = em.get( exportId, Export.class );
-
- String pathToBeParsed = config.getPath();
- String[] pathItems = pathToBeParsed.split( "/" );
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- return;
- }
-
- //update state and re-write the entity
- export.setState( Export.State.STARTED );
-
- em.update( export );
-
- Map<UUID, String> organizationGet = getOrgs(config);
- for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
- //needs to pass app name, and possibly collection to export
- exportApplicationsForOrg( organization, config, jobExecution );
- }
- export.setState( Export.State.COMPLETED );
- em.update( export );
- }
-
- private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
- // Loop through the organizations
- // TODO:this will come from the orgs in schedule when you do the validations. delete orgId
- UUID orgId = null;
-
- Map<UUID, String> organizationNames = null;
-
- if ( orgId == null ) {
- organizationNames = managementService.getOrganizations();
- }
-
- else {
- OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
-
- if ( info == null ) {
- logger.error( "Organization info is null!" );
- }
-
- organizationNames = new HashMap<UUID, String>();
- organizationNames.put( orgId, info.getName() );
- }
-
-
- return organizationNames;
- }
-
-
- public SchedulerService getSch() {
- return sch;
- }
-
-
- public void setSch( final SchedulerService sch ) {
- this.sch = sch;
- }
-
-
- public EntityManagerFactory getEmf() {
- return emf;
- }
-
-
- public void setEmf( final EntityManagerFactory emf ) {
- this.emf = emf;
- }
-
-
- public ManagementService getManagementService() {
-
- return managementService;
- }
-
- public void setManagementService( final ManagementService managementService ) {
- this.managementService = managementService;
- }
-
- //write test checking to see what happens if the input stream is closed or wrong.
- //TODO: make multipart streaming functional
- //currently only stores the collection in memory then flushes it.
- private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
- final JobExecution jobExecution ) throws Exception {
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- logger.info( "" + organization );
-
- // Loop through the applications per organization
- BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organization.getKey() );
- for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
-
- logger.info( application.getValue() + " : " + application.getKey() );
-
- String appFileName = prepareOutputFileName( "application", application.getValue() );
-
- JsonGenerator jg = getJsonGenerator( baos );
-
- EntityManager em = emf.getEntityManager( application.getKey() );
-
- jg.writeStartArray();
-
- Map<String, Object> metadata = em.getApplicationCollectionMetadata();
- long starting_time = System.currentTimeMillis();
-
- // Loop through the collections. This is the only way to loop
- // through the entities in the application (former namespace).
- //could support queries, just need to implement that in the rest endpoint.
- for ( String collectionName : metadata.keySet() ) {
- if(collectionName.equals( "exports" )) {
- continue;
- }
-
- Query query = new Query();
- query.setLimit( MAX_ENTITY_FETCH );
- query.setResultsLevel( Results.Level.ALL_PROPERTIES );
- Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
-
- PagingResultsIterator itr = new PagingResultsIterator( entities );
-
- for( Object e: itr){
- starting_time = checkTimeDelta( starting_time, jobExecution );
- Entity entity = ( Entity ) e;
-
- jg.writeStartObject();
- jg.writeFieldName( "Metadata" );
- jg.writeObject( entity );
- saveCollectionMembers( jg, em, application.getValue(), entity );
- jg.writeEndObject();
- }
- }
-
- // Close writer and file for this application.
- jg.writeEndArray();
- jg.close();
- baos.flush();
- baos.close();
-
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
- s3Export.copyToS3( is, config , appFileName);
- }
- }
-
-
- public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) {
-
- long cur_time = System.currentTimeMillis();
-
- if ( startingTime <= ( cur_time - TIMESTAMP_DELTA ) ) {
- jobExecution.heartbeat();
- return cur_time;
- }
- return startingTime;
- }
-
-
- /**
- * Serialize and save the collection members of this <code>entity</code>
- *
- * @param em Entity Manager
- * @param application Application name
- * @param entity entity
- */
- private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity )
- throws Exception {
-
- Set<String> collections = em.getCollections( entity );
- //jg.writeStartObject();
-
- // Only create entry for Entities that have collections
- if ( ( collections == null ) || collections.isEmpty() ) {
- return;
- }
-
-
- for ( String collectionName : collections ) {
-
- jg.writeFieldName( collectionName );
- jg.writeStartArray();
-
- //is 100000 an arbitary number?
- Results collectionMembers =
- em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
-
- List<UUID> entityIds = collectionMembers.getIds();
-
- if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
- for ( UUID childEntityUUID : entityIds ) {
- jg.writeObject( childEntityUUID.toString() );
- }
- }
-
- // End collection array.
- jg.writeEndArray();
- }
-
- // Write connections
- saveConnections( entity, em, jg );
-
- // Write dictionaries
- saveDictionaries( entity, em, jg );
-
- // End the object if it was Started
- //jg.writeEndObject();
- }
-
-
- /**
- * Persists the connection for this entity.
- */
- private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
-
- jg.writeFieldName( "dictionaries" );
- jg.writeStartObject();
-
- Set<String> dictionaries = em.getDictionaries( entity );
- for ( String dictionary : dictionaries ) {
-
- Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
-
- // nothing to do
- if ( dict.isEmpty() ) {
- continue;
- }
-
- jg.writeFieldName( dictionary );
-
- jg.writeStartObject();
-
- for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
- jg.writeFieldName( entry.getKey().toString() );
- jg.writeObject( entry.getValue() );
- }
-
- jg.writeEndObject();
- }
- jg.writeEndObject();
- }
-
-
- /**
- * Persists the connection for this entity.
- */
- private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
-
- jg.writeFieldName( "connections" );
- jg.writeStartObject();
-
- Set<String> connectionTypes = em.getConnectionTypes( entity );
- for ( String connectionType : connectionTypes ) {
-
- jg.writeFieldName( connectionType );
- jg.writeStartArray();
-
- Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Results.Level.IDS );
- List<ConnectionRef> connections = results.getConnections();
-
- for ( ConnectionRef connectionRef : connections ) {
- jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
- }
-
- jg.writeEndArray();
- }
- jg.writeEndObject();
- }
-
-
- protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
- //TODO:shouldn't the below be UTF-16?
- //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
-
- JsonGenerator jg = jsonFactory.createJsonGenerator( out );
- jg.setPrettyPrinter( new DefaultPrettyPrinter() );
- jg.setCodec( new ObjectMapper() );
- return jg;
- }
-
-
- protected File createOutputFile( String type, String name ) {
- return new File( prepareOutputFileName( type, name ) );
- }
-
-
- /**
- * @param type just a label such us: organization, application.
- *
- * @return the file name concatenated with the type and the name of the collection
- */
- protected String prepareOutputFileName( String type, String name ) {
- StringBuilder str = new StringBuilder();
- str.append( name );
- str.append( "." );
- str.append( startTime );
- str.append( ".json" );
-
- String outputFileName = str.toString();
- //TODO:this is , i feel, bad practice so make sure to come back here and fix it up.
-
- return outputFileName;
- }
-
-
- @Autowired
- @Override
- public void setS3Export( S3Export s3Export ) { this.s3Export = s3Export; }
-}
[3/9] Added classes to the branch that were labeled untracked.
Posted by sn...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
deleted file mode 100644
index e61d64c..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/S3Export.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.io.InputStream;
-
-import org.usergrid.management.ExportInfo;
-
-
-/**
- *
- *
- */
-public interface S3Export {
- void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename );
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
deleted file mode 100644
index c574e01..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/S3ExportImpl.java
+++ /dev/null
@@ -1,84 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.io.InputStream;
-import java.util.Properties;
-
-import org.jclouds.ContextBuilder;
-import org.jclouds.blobstore.AsyncBlobStore;
-import org.jclouds.blobstore.BlobStoreContext;
-import org.jclouds.blobstore.domain.Blob;
-import org.jclouds.blobstore.domain.BlobBuilder;
-import org.jclouds.blobstore.options.PutOptions;
-import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
-import org.jclouds.logging.log4j.config.Log4JLoggingModule;
-import org.jclouds.netty.config.NettyPayloadModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.usergrid.management.ExportInfo;
-
-import com.google.common.collect.ImmutableSet;
-import com.google.common.util.concurrent.ListenableFuture;
-import com.google.inject.Module;
-
-
-/**
- *
- *
- */
-public class S3ExportImpl implements S3Export {
-
- @Override
- public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
-
- Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
- /*won't need any of the properties as I have the export info*/
- String bucketName = exportInfo.getBucket_location();
- String accessId = exportInfo.getS3_accessId();
- String secretKey = exportInfo.getS3_key();
-
- Properties overrides = new Properties();
- overrides.setProperty( "s3" + ".identity", accessId );
- overrides.setProperty( "s3" + ".credential", secretKey );
-
- final Iterable<? extends Module> MODULES = ImmutableSet
- .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule
- () );
-
- BlobStoreContext context =
- ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
- .overrides( overrides ).buildView( BlobStoreContext.class );
-
- // Create Container (the bucket in s3)
- try {
- AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
- // BlobStore (returns false if it already exists)
- ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName );
- if ( container.get() ) {
- logger.info( "Created bucket " + bucketName );
- }
- }
- catch ( Exception ex ) {
- logger.error( "Could not start binary service: {}", ex.getMessage() );
- return;
- }
-
- try {
- AsyncBlobStore blobStore = context.getAsyncBlobStore();
- BlobBuilder blobBuilder =
- blobStore.blobBuilder( filename ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
-
-
- Blob blob = blobBuilder.build();
-
- ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
-
- logger.info( "Uploaded file etag=" + futureETag.get() );
- }
- catch ( Exception e ) {
- logger.error( "Error uploading to blob store", e );
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index d663890..ead98e1 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -65,12 +65,12 @@
<property name="managementService" ref="managementService"/>
</bean>
- <bean id="exportService" class="org.usergrid.management.export.ExportServiceImpl" >
+ <bean id="exportService" class="org.apache.usergrid.management.export.ExportServiceImpl" >
<property name="managementService" ref="managementService"/>
<property name="emf" ref="entityManagerFactory"/>
<property name="sch" ref="schedulerService"/>
</bean>
- <bean id="s3Export" class="org.usergrid.management.export.S3ExportImpl" />
+ <bean id="s3Export" class="org.apache.usergrid.management.export.S3ExportImpl" />
</beans>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
index 12e2ab0..4df8e5e 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetup.java
@@ -5,6 +5,7 @@ import java.util.Properties;
import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory;
@@ -21,6 +22,8 @@ public interface ServiceITSetup extends CoreITSetup {
Properties getProps();
+ ExportService getExportService();
+
/**
* Convenience method to set a property in the Properties object returned by getProps();
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
index 6008610..e200b4f 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
@@ -11,6 +11,7 @@ import org.springframework.beans.factory.config.PropertiesFactoryBean;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.export.ExportService;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
@@ -27,6 +28,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
private TokenService tokenService;
private SignInProviderFactory providerFactory;
private Properties properties;
+ private ExportService exportService;
public ServiceITSetupImpl( CassandraResource cassandraResource ) {
@@ -48,6 +50,7 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
providerFactory = cassandraResource.getBean( SignInProviderFactory.class );
properties = cassandraResource.getBean( PropertiesFactoryBean.class ).getObject();
smf = cassandraResource.getBean( ServiceManagerFactory.class );
+ exportService = cassandraResource.getBean( ExportService.class );
LOG.info( "Test setup complete..." );
}
@@ -82,6 +85,9 @@ public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetu
return managementService;
}
+ @Override
+ public ExportService getExportService() { return exportService; }
+
public ServiceManagerFactory getSmf() {
if ( smf == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index 99b2ee9..e1b297c 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -1,20 +1,35 @@
package org.apache.usergrid.management.cassandra;
+import java.io.File;
+import java.io.FileReader;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import org.json.simple.JSONObject;
+import org.json.simple.parser.JSONParser;
import org.junit.BeforeClass;
import org.junit.ClassRule;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.management.export.ExportJob;
+import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.management.export.S3Export;
+import org.apache.usergrid.management.export.S3ExportImpl;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.Export;
+
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
import org.apache.usergrid.ServiceITSuite;
+import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.cassandra.Concurrent;
@@ -23,6 +38,7 @@ import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.entities.JobData;
import org.apache.usergrid.persistence.entities.User;
import org.apache.usergrid.security.AuthPrincipalType;
import org.apache.usergrid.security.crypto.command.Md5HashCommand;
@@ -42,6 +58,8 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/** @author zznate */
@@ -726,4 +744,408 @@ public class ManagementServiceIT {
assertEquals( userId, authedUser.getUuid() );
}
+ //Tests to make sure we can call the job with mock data and it runs.
+ @Ignore
+ public void testFileConnections() throws Exception {
+
+ File f = null;
+
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[10];
+ //creates entities
+ for (int i = 0; i< 10;i++) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+
+ entity[i] = em.create( "user", userProperties );
+
+ }
+ //creates connections
+ em.createConnection( em.getRef( entity[0].getUuid() ),"Vibrations",em.getRef( entity[1].getUuid() ) );
+ em.createConnection( em.getRef( entity[1].getUuid() ),"Vibrations",em.getRef( entity[0].getUuid() ) );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ //assertEquals(13, a.size() );
+
+ org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 0 );
+ org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject) objEnt.get( "connections" );
+
+ assertNotNull( objConnections );
+
+ org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get("Vibrations");
+
+ assertNotNull( objVibrations );
+
+ f.delete();
+ }
+
+
+ @Test
+ public void testFileValidity() throws Exception {
+
+ File f = null;
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
+ assertNotNull( entityData );
+
+ }
+ f.delete();
+ }
+
+ @Test
+ public void testFileExportOneOrg() throws Exception {
+
+ File f = null;
+
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
+ }
+ setup.getMgmtSvc().createOwnerAndOrganization( "noExport","junkUserName","junkRealName","ugExport@usergrid.com","123456789" );
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+
+ //assertEquals( 3 , a.size() );
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
+ String entityName = ( String) entityData.get( "name" );
+ // assertNotEquals( "NotEqual","junkRealName",entityName );
+ assertFalse( "junkRealName".equals( entityName ) );
+
+ }
+ f.delete();
+ }
+ @Test
+ public void testFileExportOneApp() throws Exception {
+
+ File f = null;
+ String orgName = "ed-organization";
+ String appName = "testAppNotExported";
+
+ try {
+ f = new File ("test.json");
+ f.delete();
+ } catch (Exception e) {
+ //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
+ }
+
+ UUID appId = setup.getEmf().createApplication( orgName, appName );
+
+
+ EntityManager em = setup.getEmf().getEntityManager( appId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[10];
+ //creates entities
+ for (int i = 0; i< 10;i++) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ entity[i] = em.create( "user", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+
+ //assertEquals( 3 , a.size() );
+ for (int i = 0; i < a.size();i++ )
+ {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
+ String entityName = ( String) entityData.get( "name" );
+ assertFalse( "junkRealName".equals( entityName ) );
+ //assertNotEquals( "NotEquals","junkRealName",entityName );
+ }
+ f.delete();
+ }
+
+ //only handles the DoJob Code , different tests for DoExport
+ @Test
+ public void testExportDoJob() throws Exception {
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo ); //this needs to be populated with properties of export info
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ ExportService eS = mock (ExportService.class);
+ job.setExportService( eS );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+
+ }
+
+ @Test
+ public void testExportDoExport() throws Exception {
+
+ EntityManagerFactory emf = setup.getEmf();
+ EntityManager em = emf.getEntityManager( applicationId );
+ HashMap<String, Object> payload = payloadBuilder();
+ ExportService eS = setup.getExportService();
+
+ JobExecution jobExecution = mock( JobExecution.class);
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ UUID entityExportUUID = eS.schedule( exportInfo );
+
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", entityExportUUID);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ //Exportem.get(entityExport);
+ Export exportEntity = ( Export ) em.get(entityExportUUID);
+ assertNotNull( exportEntity );
+ String derp = exportEntity.getState().name();
+ assertEquals( "PENDING",exportEntity.getState().name());
+ try {
+ eS.doExport( exportInfo,jobExecution );
+ }catch(Exception e) {
+ throw e;
+ //assert(false);
+ }
+ exportEntity = ( Export ) em.get(entityExportUUID);
+ assertNotNull( exportEntity );
+ assertEquals( "COMPLETED",exportEntity.getState().name() );
+ }
+
+ //tests that with empty job data, the export still runs.
+ @Test
+ public void testExportEmptyJobData() throws Exception {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+ }
+
+ @Test
+ public void testNullJobExecution () {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }catch ( Exception e) {
+ assert( false );
+ }
+ assert(true);
+ }
+
+ @Ignore //For this test please input your s3 credentials into payload builder.
+ public void testIntegration100Entities() throws Exception {
+
+ S3Export s3Export = new S3ExportImpl();
+ ExportService exportService = setup.getExportService();
+ HashMap<String, Object> payload = payloadBuilder();
+
+ ExportInfo exportInfo = new ExportInfo(payload);
+ exportInfo.setApplicationId( applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[100];
+ //creates entities
+ for (int i = 0; i< 100;i++) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+
+ entity[i] = em.create( "user", userProperties );
+
+ }
+
+ UUID exportUUID = exportService.schedule( exportInfo );
+ exportService.setS3Export( s3Export );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", exportInfo );
+ jobData.setProperty( "exportId", exportUUID );
+
+ JobExecution jobExecution = mock ( JobExecution.class);
+ when(jobExecution.getJobData()).thenReturn( jobData );
+
+ exportService.doExport( exportInfo, jobExecution );
+
+ }
+
+
+ /*Creates fake payload for testing purposes.*/
+ public HashMap<String,Object> payloadBuilder() {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ // TODO: always put dummy values here and ignore this test.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app");
+ payload.put( "properties", properties);
+ return payload;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
new file mode 100644
index 0000000..432fe5e
--- /dev/null
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -0,0 +1,49 @@
+package org.apache.usergrid.management.cassandra;
+
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.management.export.S3Export;
+
+
+/**
+ * Writes to file instead of s3.
+ *
+ */
+public class MockS3ExportImpl implements S3Export {
+ @Override
+ public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
+ Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
+ int read = 0;
+ byte[] bytes = new byte[1024];
+ OutputStream outputStream = null;
+ //FileInputStream fis = new PrintWriter( inputStream );
+
+ try {
+ outputStream = new FileOutputStream( new File("test.json") );
+
+ }
+ catch ( FileNotFoundException e ) {
+ e.printStackTrace();
+ }
+
+
+ try {
+ while ( (read = (inputStream.read( bytes ))) != -1) {
+ outputStream.write( bytes, 0, read );
+ }
+
+ }
+ catch ( IOException e ) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/ServiceITSetup.java b/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
deleted file mode 100644
index 7cd370a..0000000
--- a/stack/services/src/test/java/org/usergrid/ServiceITSetup.java
+++ /dev/null
@@ -1,47 +0,0 @@
-package org.usergrid;
-
-
-import java.util.Properties;
-
-import org.usergrid.management.ApplicationCreator;
-import org.usergrid.management.ManagementService;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.security.providers.SignInProviderFactory;
-import org.usergrid.security.tokens.TokenService;
-import org.usergrid.services.ServiceManagerFactory;
-
-
-public interface ServiceITSetup extends CoreITSetup {
- ManagementService getMgmtSvc();
-
- ApplicationCreator getAppCreator();
-
- ServiceManagerFactory getSmf();
-
- TokenService getTokenSvc();
-
- Properties getProps();
-
- ExportService getExportService();
-
- /**
- * Convenience method to set a property in the Properties object returned by getProps();
- *
- * @param key the property key
- * @param value the value of the property to set
- *
- * @return the previous value of the property
- */
- Object set( String key, String value );
-
- /**
- * Convenience method to get a property in the Properties object returned by getProps().
- *
- * @param key the property key
- *
- * @return value the value of the property
- */
- String get( String key );
-
- SignInProviderFactory getProviderFactory();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java b/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
deleted file mode 100644
index 026b61d..0000000
--- a/stack/services/src/test/java/org/usergrid/ServiceITSetupImpl.java
+++ /dev/null
@@ -1,135 +0,0 @@
-package org.usergrid;
-
-
-import java.util.Properties;
-
-import org.junit.runner.Description;
-import org.junit.runners.model.Statement;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.config.PropertiesFactoryBean;
-import org.usergrid.cassandra.CassandraResource;
-import org.usergrid.management.ApplicationCreator;
-import org.usergrid.management.ManagementService;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.persistence.cassandra.CassandraService;
-import org.usergrid.security.providers.SignInProviderFactory;
-import org.usergrid.security.tokens.TokenService;
-import org.usergrid.services.ServiceManagerFactory;
-
-
-/** A {@link org.junit.rules.TestRule} that sets up services. */
-public class ServiceITSetupImpl extends CoreITSetupImpl implements ServiceITSetup {
- private static final Logger LOG = LoggerFactory.getLogger( ServiceITSetupImpl.class );
-
- private ServiceManagerFactory smf;
- private ManagementService managementService;
- private ApplicationCreator applicationCreator;
- private TokenService tokenService;
- private SignInProviderFactory providerFactory;
- private Properties properties;
- private ExportService exportService;
-
-
- public ServiceITSetupImpl( CassandraResource cassandraResource ) {
- super( cassandraResource );
- }
-
-
- protected void after( Description description ) {
- super.after( description );
- LOG.info( "Test {}: finish with application", description.getDisplayName() );
- }
-
-
- protected void before( Description description ) throws Throwable {
- super.before( description );
- managementService = cassandraResource.getBean( ManagementService.class );
- applicationCreator = cassandraResource.getBean( ApplicationCreator.class );
- tokenService = cassandraResource.getBean( TokenService.class );
- providerFactory = cassandraResource.getBean( SignInProviderFactory.class );
- properties = cassandraResource.getBean( PropertiesFactoryBean.class ).getObject();
- smf = cassandraResource.getBean( ServiceManagerFactory.class );
- exportService = cassandraResource.getBean( ExportService.class );
-
- LOG.info( "Test setup complete..." );
- }
-
-
- @Override
- public Statement apply( final Statement base, final Description description ) {
- return new Statement() {
- @Override
- public void evaluate() throws Throwable {
- before( description );
-
- try {
- base.evaluate();
- }
- finally {
- after( description );
- }
- }
- };
- }
-
-
- @Override
- public CassandraService getCassSvc() {
- return cassandraResource.getBean( CassandraService.class );
- }
-
-
- @Override
- public ManagementService getMgmtSvc() {
- return managementService;
- }
-
-
- public ServiceManagerFactory getSmf() {
- if ( smf == null ) {
- smf = cassandraResource.getBean( ServiceManagerFactory.class );
- }
-
- return smf;
- }
-
-
- @Override
- public ApplicationCreator getAppCreator() {
- return applicationCreator;
- }
-
-
- @Override
- public TokenService getTokenSvc() {
- return tokenService;
- }
-
-
- @Override
- public Properties getProps() {
- return properties;
- }
-
- @Override
- public ExportService getExportService() { return exportService; }
-
-
- @Override
- public Object set( String key, String value ) {
- return properties.setProperty( key, value );
- }
-
-
- @Override
- public String get( String key ) {
- return properties.getProperty( key );
- }
-
-
- @Override
- public SignInProviderFactory getProviderFactory() {
- return providerFactory;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
deleted file mode 100644
index 41f6259..0000000
--- a/stack/services/src/test/java/org/usergrid/management/cassandra/ManagementServiceIT.java
+++ /dev/null
@@ -1,1152 +0,0 @@
-package org.usergrid.management.cassandra;
-
-
-import java.io.File;
-import java.io.FileReader;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
-import org.junit.BeforeClass;
-import org.junit.ClassRule;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.usergrid.ServiceITSetup;
-import org.usergrid.ServiceITSetupImpl;
-import org.usergrid.ServiceITSuite;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.cassandra.CassandraResource;
-import org.usergrid.cassandra.ClearShiroSubject;
-import org.usergrid.cassandra.Concurrent;
-import org.usergrid.count.SimpleBatcher;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.management.UserInfo;
-import org.usergrid.management.export.ExportJob;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.management.export.S3Export;
-import org.usergrid.management.export.S3ExportImpl;
-import org.usergrid.persistence.CredentialsInfo;
-import org.usergrid.persistence.Entity;
-import org.usergrid.persistence.EntityManager;
-import org.usergrid.persistence.EntityManagerFactory;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.persistence.entities.JobData;
-import org.usergrid.persistence.entities.User;
-import org.usergrid.security.AuthPrincipalType;
-import org.usergrid.security.crypto.command.Md5HashCommand;
-import org.usergrid.security.crypto.command.Sha1HashCommand;
-import org.usergrid.security.tokens.TokenCategory;
-import org.usergrid.security.tokens.exceptions.InvalidTokenException;
-import org.usergrid.utils.JsonUtils;
-import org.usergrid.utils.UUIDUtils;
-
-import org.apache.usergrid.management.cassandra.MockS3ExportImpl;
-
-import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
-import static org.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
-
-/** @author zznate */
-@Concurrent()
-public class ManagementServiceIT {
- private static final Logger LOG = LoggerFactory.getLogger( ManagementServiceIT.class );
-
- private static CassandraResource cassandraResource = ServiceITSuite.cassandraResource;
-
- // app-level data generated only once
- private static UserInfo adminUser;
- private static OrganizationInfo organization;
- private static UUID applicationId;
-
- @Rule
- public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
-
- @ClassRule
- public static final ServiceITSetup setup = new ServiceITSetupImpl( cassandraResource );
-
-
- @BeforeClass
- public static void setup() throws Exception {
- LOG.info( "in setup" );
- adminUser = setup.getMgmtSvc().createAdminUser( "edanuff", "Ed Anuff", "ed@anuff.com", "test", false, false );
- organization = setup.getMgmtSvc().createOrganization( "ed-organization", adminUser, true );
- applicationId = setup.getMgmtSvc().createApplication( organization.getUuid(), "ed-application" ).getId();
- }
-
-
- @Test
- public void testGetTokenForPrincipalAdmin() throws Exception {
- String token = ( ( ManagementServiceImpl ) setup.getMgmtSvc() )
- .getTokenForPrincipal( TokenCategory.ACCESS, null, MANAGEMENT_APPLICATION_ID,
- AuthPrincipalType.ADMIN_USER, adminUser.getUuid(), 0 );
- // ^ same as:
- // managementService.getAccessTokenForAdminUser(user.getUuid());
- assertNotNull( token );
- token = ( ( ManagementServiceImpl ) setup.getMgmtSvc() )
- .getTokenForPrincipal( TokenCategory.ACCESS, null, MANAGEMENT_APPLICATION_ID,
- AuthPrincipalType.APPLICATION_USER, adminUser.getUuid(), 0 );
- // This works because ManagementService#getSecret takes the same code
- // path
- // on an OR for APP._USER as for ADMIN_USER
- // is ok technically as ADMIN_USER is a APP_USER to the admin app, but
- // should still
- // be stricter checking
- assertNotNull( token );
- // managementService.getTokenForPrincipal(appUuid, authPrincipal, pUuid,
- // salt, true);
- }
-
-
- @Test
- public void testGetTokenForPrincipalUser() throws Exception {
- // create a user
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "edanuff" );
- properties.put( "email", "ed@anuff.com" );
-
- Entity user = setup.getEmf().getEntityManager( applicationId ).create( "user", properties );
-
- assertNotNull( user );
- String token = ( ( ManagementServiceImpl ) setup.getMgmtSvc() )
- .getTokenForPrincipal( TokenCategory.ACCESS, null, MANAGEMENT_APPLICATION_ID,
- AuthPrincipalType.APPLICATION_USER, user.getUuid(), 0 );
- assertNotNull( token );
- }
-
-
- @Test
- public void testCountAdminUserAction() throws Exception {
- SimpleBatcher batcher = cassandraResource.getBean( SimpleBatcher.class );
-
- batcher.setBlockingSubmit( true );
- batcher.setBatchSize( 1 );
-
- setup.getMgmtSvc().countAdminUserAction( adminUser, "login" );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- Map<String, Long> counts = em.getApplicationCounters();
- LOG.info( JsonUtils.mapToJsonString( counts ) );
- LOG.info( JsonUtils.mapToJsonString( em.getCounterNames() ) );
- assertNotNull( counts.get( "admin_logins" ) );
- assertEquals( 1, counts.get( "admin_logins" ).intValue() );
- }
-
-
- @Test
- public void deactivateUser() throws Exception {
-
- UUID uuid = UUIDUtils.newTimeUUID();
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "test" + uuid );
- properties.put( "email", String.format( "test%s@anuff.com", uuid ) );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
- Entity entity = em.create( "user", properties );
-
- assertNotNull( entity );
-
- User user = em.get( entity.getUuid(), User.class );
-
- assertFalse( user.activated() );
- assertNull( user.getDeactivated() );
-
- setup.getMgmtSvc().activateAppUser( applicationId, user.getUuid() );
-
- user = em.get( entity.getUuid(), User.class );
-
- assertTrue( user.activated() );
- assertNull( user.getDeactivated() );
-
- // get a couple of tokens. These shouldn't work after we deactive the user
- String token1 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, user.getUuid(), 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, user.getUuid(), 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- long startTime = System.currentTimeMillis();
-
- setup.getMgmtSvc().deactivateUser( applicationId, user.getUuid() );
-
- long endTime = System.currentTimeMillis();
-
- user = em.get( entity.getUuid(), User.class );
-
- assertFalse( user.activated() );
- assertNotNull( user.getDeactivated() );
-
- assertTrue( startTime <= user.getDeactivated() && user.getDeactivated() <= endTime );
-
- boolean invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
-
- invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
- }
-
-
- @Test
- public void disableAdminUser() throws Exception {
-
- UUID uuid = UUIDUtils.newTimeUUID();
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "test" + uuid );
- properties.put( "email", String.format( "test%s@anuff.com", uuid ) );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- Entity entity = em.create( "user", properties );
-
- assertNotNull( entity );
-
- User user = em.get( entity.getUuid(), User.class );
-
- assertFalse( user.activated() );
- assertNull( user.getDeactivated() );
-
- setup.getMgmtSvc().activateAdminUser( user.getUuid() );
-
- user = em.get( entity.getUuid(), User.class );
-
- assertTrue( user.activated() );
- assertNull( user.getDeactivated() );
-
- // get a couple of tokens. These shouldn't work after we deactive the user
- String token1 = setup.getMgmtSvc().getAccessTokenForAdminUser( user.getUuid(), 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAdminUser( user.getUuid(), 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().disableAdminUser( user.getUuid() );
-
- user = em.get( entity.getUuid(), User.class );
-
- assertTrue( user.disabled() );
-
- boolean invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
-
- invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
- }
-
-
- @Test
- public void userTokensRevoke() throws Exception {
- UUID userId = UUIDUtils.newTimeUUID();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokensForAppUser( applicationId, userId );
-
- boolean invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
-
- invalidTokenExcpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenExcpetion = true;
- }
-
- assertTrue( invalidTokenExcpetion );
- }
-
-
- @Test
- public void userTokenRevoke() throws Exception {
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
-
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "realbeast" );
- properties.put( "email", "sungju@softwaregeeks.org" );
-
- Entity user = em.create( "user", properties );
- assertNotNull( user );
-
- UUID userId = user.getUuid();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAppUser( applicationId, userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokenForAppUser( token1 );
-
- boolean invalidToken1Excpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken1Excpetion = true;
- }
-
- assertTrue( invalidToken1Excpetion );
-
- boolean invalidToken2Excpetion = true;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken2Excpetion = false;
- }
-
- assertTrue( invalidToken2Excpetion );
- }
-
-
- @Test
- public void adminTokensRevoke() throws Exception {
- UUID userId = UUIDUtils.newTimeUUID();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokensForAdminUser( userId );
-
- boolean invalidTokenException = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenException = true;
- }
-
- assertTrue( invalidTokenException );
-
- invalidTokenException = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidTokenException = true;
- }
-
- assertTrue( invalidTokenException );
- }
-
-
- @Test
- public void adminTokenRevoke() throws Exception {
- UUID userId = adminUser.getUuid();
-
- String token1 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
- String token2 = setup.getMgmtSvc().getAccessTokenForAdminUser( userId, 0 );
-
- assertNotNull( setup.getTokenSvc().getTokenInfo( token1 ) );
- assertNotNull( setup.getTokenSvc().getTokenInfo( token2 ) );
-
- setup.getMgmtSvc().revokeAccessTokenForAdminUser( userId, token1 );
-
- boolean invalidToken1Excpetion = false;
-
- try {
- setup.getTokenSvc().getTokenInfo( token1 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken1Excpetion = true;
- }
-
- assertTrue( invalidToken1Excpetion );
-
- boolean invalidToken2Excpetion = true;
-
- try {
- setup.getTokenSvc().getTokenInfo( token2 );
- }
- catch ( InvalidTokenException ite ) {
- invalidToken2Excpetion = false;
- }
-
- assertTrue( invalidToken2Excpetion );
- }
-
-
- @Test
- public void superUserGetOrganizationsPage() throws Exception {
- int beforeSize = setup.getMgmtSvc().getOrganizations().size() - 1;
- // create 15 orgs
- for ( int x = 0; x < 15; x++ ) {
- setup.getMgmtSvc().createOrganization( "super-user-org-" + x, adminUser, true );
- }
- // should be 17 total
- assertEquals( 16 + beforeSize, setup.getMgmtSvc().getOrganizations().size() );
- List<OrganizationInfo> orgs = setup.getMgmtSvc().getOrganizations( null, 10 );
- assertEquals( 10, orgs.size() );
- UUID val = orgs.get( 9 ).getUuid();
- orgs = setup.getMgmtSvc().getOrganizations( val, 10 );
- assertEquals( 7 + beforeSize, orgs.size() );
- assertEquals( val, orgs.get( 0 ).getUuid() );
- }
-
-
- @Test
- public void authenticateAdmin() throws Exception {
-
- String username = "tnine";
- String password = "test";
-
- UserInfo adminUser = setup.getMgmtSvc()
- .createAdminUser( username, "Todd Nine", UUID.randomUUID() + "@apigee.com", password,
- false, false );
-
- UserInfo authedUser = setup.getMgmtSvc().verifyAdminUserPasswordCredentials( username, password );
-
- assertEquals( adminUser.getUuid(), authedUser.getUuid() );
-
- authedUser = setup.getMgmtSvc().verifyAdminUserPasswordCredentials( adminUser.getEmail(), password );
-
- assertEquals( adminUser.getUuid(), authedUser.getUuid() );
-
- authedUser = setup.getMgmtSvc().verifyAdminUserPasswordCredentials( adminUser.getUuid().toString(), password );
-
- assertEquals( adminUser.getUuid(), authedUser.getUuid() );
- }
-
-
- /** Test we can change the password if it's hashed with sha1 */
- @Test
- public void testAdminPasswordChangeShaType() throws Exception {
- String username = "testAdminPasswordChangeShaType";
- String password = "test";
-
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Sha1HashCommand command = new Sha1HashCommand();
- byte[] hashed = command.hash( password.getBytes( "UTF-8" ), info, userId, MANAGEMENT_APPLICATION_ID );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- info.setCipher( command.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( MANAGEMENT_APPLICATION_ID, userId, password, newPassword );
-
- //verify authorization works
- authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
-
- /** Test we can change the password if it's hashed with md5 then sha1 */
- @Test
- public void testAdminPasswordChangeMd5ShaType() throws Exception {
- String username = "testAdminPasswordChangeMd5ShaType";
- String password = "test";
-
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( MANAGEMENT_APPLICATION_ID );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Md5HashCommand md5 = new Md5HashCommand();
-
- Sha1HashCommand sha1 = new Sha1HashCommand();
-
- byte[] hashed = md5.hash( password.getBytes( "UTF-8" ), info, userId, MANAGEMENT_APPLICATION_ID );
- hashed = sha1.hash( hashed, info, userId, MANAGEMENT_APPLICATION_ID );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- //set the final cipher to sha1
- info.setCipher( sha1.getName() );
- //set the next hash type to md5
- info.setHashType( md5.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( MANAGEMENT_APPLICATION_ID, userId, password, newPassword );
-
- //verify authorization works
- authedUser =
- setup.getMgmtSvc().verifyAppUserPasswordCredentials( MANAGEMENT_APPLICATION_ID, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
-
- @Test
- public void authenticateUser() throws Exception {
-
- String username = "tnine";
- String password = "test";
- String orgName = "autneticateUser";
- String appName = "authenticateUser";
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password );
-
- //verify authorization works
- User authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password, newPassword );
-
- //verify authorization works
- authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, newPassword );
- }
-
-
- /** Test we can change the password if it's hashed with sha1 */
- @Test
- public void testAppUserPasswordChangeShaType() throws Exception {
- String username = "tnine";
- String password = "test";
- String orgName = "testAppUserPasswordChangeShaType";
- String appName = "testAppUserPasswordChangeShaType";
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Sha1HashCommand command = new Sha1HashCommand();
- byte[] hashed = command.hash( password.getBytes( "UTF-8" ), info, userId, appId );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- info.setCipher( command.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password, newPassword );
-
- //verify authorization works
- authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
-
- /** Test we can change the password if it's hashed with md5 then sha1 */
- @Test
- public void testAppUserPasswordChangeMd5ShaType() throws Exception {
- String username = "tnine";
- String password = "test";
- String orgName = "testAppUserPasswordChangeMd5ShaType";
- String appName = "testAppUserPasswordChangeMd5ShaType";
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
- User user = new User();
- user.setActivated( true );
- user.setUsername( username );
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
-
- User storedUser = em.create( user );
-
-
- UUID userId = storedUser.getUuid();
-
- //set the password in the sha1 format
- CredentialsInfo info = new CredentialsInfo();
- info.setRecoverable( false );
- info.setEncrypted( true );
-
-
- Md5HashCommand md5 = new Md5HashCommand();
-
- Sha1HashCommand sha1 = new Sha1HashCommand();
-
- byte[] hashed = md5.hash( password.getBytes( "UTF-8" ), info, userId, appId );
- hashed = sha1.hash( hashed, info, userId, appId );
-
- info.setSecret( encodeBase64URLSafeString( hashed ) );
- //set the final cipher to sha1
- info.setCipher( sha1.getName() );
- //set the next hash type to md5
- info.setHashType( md5.getName() );
-
-
- em.addToDictionary( storedUser, DICTIONARY_CREDENTIALS, "password", info );
-
-
- //verify authorization works
- User authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, password );
-
- assertEquals( userId, authedUser.getUuid() );
-
- //test we can change the password
- String newPassword = "test2";
-
- setup.getMgmtSvc().setAppUserPassword( appId, userId, password, newPassword );
-
- //verify authorization works
- authedUser = setup.getMgmtSvc().verifyAppUserPasswordCredentials( appId, username, newPassword );
-
- assertEquals( userId, authedUser.getUuid() );
- }
-
- //Tests to make sure we can call the job with mock data and it runs.
- @Ignore
- public void testFileConnections() throws Exception {
-
- File f = null;
-
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
- }
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
- //intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[10];
- //creates entities
- for (int i = 0; i< 10;i++) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
-
- entity[i] = em.create( "user", userProperties );
-
- }
- //creates connections
- em.createConnection( em.getRef( entity[0].getUuid() ),"Vibrations",em.getRef( entity[1].getUuid() ) );
- em.createConnection( em.getRef( entity[1].getUuid() ),"Vibrations",em.getRef( entity[0].getUuid() ) );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- //create and initialize jobData returned in JobExecution.
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
- //assertEquals(13, a.size() );
-
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 0 );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject) objEnt.get( "connections" );
-
- assertNotNull( objConnections );
-
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get("Vibrations");
-
- assertNotNull( objVibrations );
-
- f.delete();
- }
-
-
- @Test
- public void testFileValidity() throws Exception {
-
- File f = null;
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
- }
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
-
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- assertNotNull( entityData );
-
- }
- f.delete();
- }
-
- @Test
- public void testFileExportOneOrg() throws Exception {
-
- File f = null;
-
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
- }
- setup.getMgmtSvc().createOwnerAndOrganization( "noExport","junkUserName","junkRealName","ugExport@usergrid.com","123456789" );
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
-
- //assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
- // assertNotEquals( "NotEqual","junkRealName",entityName );
- assertFalse( "junkRealName".equals( entityName ) );
-
- }
- f.delete();
- }
- @Test
- public void testFileExportOneApp() throws Exception {
-
- File f = null;
- String orgName = "ed-organization";
- String appName = "testAppNotExported";
-
- try {
- f = new File ("test.json");
- f.delete();
- } catch (Exception e) {
- //consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
- }
-
- UUID appId = setup.getEmf().createApplication( orgName, appName );
-
-
- EntityManager em = setup.getEmf().getEntityManager( appId );
- //intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[10];
- //creates entities
- for (int i = 0; i< 10;i++) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
- entity[i] = em.create( "user", userProperties );
- }
-
- S3Export s3Export = new MockS3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- JSONParser parser = new JSONParser();
-
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
-
- //assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject) a.get( i );
- org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
- assertFalse( "junkRealName".equals( entityName ) );
- //assertNotEquals( "NotEquals","junkRealName",entityName );
- }
- f.delete();
- }
-
- //only handles the DoJob Code , different tests for DoExport
- @Test
- public void testExportDoJob() throws Exception {
-
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo ); //this needs to be populated with properties of export info
-
- JobExecution jobExecution = mock ( JobExecution.class);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- ExportJob job = new ExportJob();
- ExportService eS = mock (ExportService.class);
- job.setExportService( eS );
- try {
- job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
- }
- assert(true);
-
- }
-
- @Test
- public void testExportDoExport() throws Exception {
-
- EntityManagerFactory emf = setup.getEmf();
- EntityManager em = emf.getEntityManager( applicationId );
- HashMap<String, Object> payload = payloadBuilder();
- ExportService eS = setup.getExportService();
-
- JobExecution jobExecution = mock( JobExecution.class);
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- UUID entityExportUUID = eS.schedule( exportInfo );
-
-
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", entityExportUUID);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- //Exportem.get(entityExport);
- Export exportEntity = ( Export ) em.get(entityExportUUID);
- assertNotNull( exportEntity );
- String derp = exportEntity.getState().name();
- assertEquals( "PENDING",exportEntity.getState().name());
- try {
- eS.doExport( exportInfo,jobExecution );
- }catch(Exception e) {
- throw e;
- //assert(false);
- }
- exportEntity = ( Export ) em.get(entityExportUUID);
- assertNotNull( exportEntity );
- assertEquals( "COMPLETED",exportEntity.getState().name() );
- }
-
- //tests that with empty job data, the export still runs.
- @Test
- public void testExportEmptyJobData() throws Exception {
-
- JobData jobData = new JobData();
-
- JobExecution jobExecution = mock ( JobExecution.class);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- ExportJob job = new ExportJob();
- S3Export s3Export = mock( S3Export.class );
- setup.getExportService().setS3Export( s3Export );
- job.setExportService( setup.getExportService() );
- try {
- job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
- }
- assert(true);
- }
-
- @Test
- public void testNullJobExecution () {
-
- JobData jobData = new JobData();
-
- JobExecution jobExecution = mock ( JobExecution.class);
-
- when( jobExecution.getJobData() ).thenReturn( jobData );
-
- ExportJob job = new ExportJob();
- S3Export s3Export = mock( S3Export.class );
- setup.getExportService().setS3Export( s3Export );
- job.setExportService( setup.getExportService() );
- try {
- job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
- }
- assert(true);
- }
-
- @Ignore //For this test please input your s3 credentials into payload builder.
- public void testIntegration100Entities() throws Exception {
-
- S3Export s3Export = new S3ExportImpl();
- ExportService exportService = setup.getExportService();
- HashMap<String, Object> payload = payloadBuilder();
-
- ExportInfo exportInfo = new ExportInfo(payload);
- exportInfo.setApplicationId( applicationId );
-
- EntityManager em = setup.getEmf().getEntityManager( applicationId );
- //intialize user object to be posted
- Map<String, Object> userProperties = null;
- Entity[] entity;
- entity = new Entity[100];
- //creates entities
- for (int i = 0; i< 100;i++) {
- userProperties = new LinkedHashMap<String, Object>();
- userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
-
- entity[i] = em.create( "user", userProperties );
-
- }
-
- UUID exportUUID = exportService.schedule( exportInfo );
- exportService.setS3Export( s3Export );
-
- //create and initialize jobData returned in JobExecution.
- JobData jobData = new JobData();
- jobData.setProperty( "jobName", "exportJob" );
- jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", exportUUID );
-
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
-
- }
-
-
- /*Creates fake payload for testing purposes.*/
- public HashMap<String,Object> payloadBuilder() {
- HashMap<String, Object> payload = new HashMap<String, Object>();
- Map<String, Object> properties = new HashMap<String, Object>();
- Map<String, Object> storage_info = new HashMap<String, Object>();
-// TODO: always put dummy values here and ignore this test.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
-
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
-
- payload.put( "path", "test-organization/test-app");
- payload.put( "properties", properties);
- return payload;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java
deleted file mode 100644
index f66b67d..0000000
--- a/stack/services/src/test/java/org/usergrid/management/cassandra/MockS3ExportImpl.java
+++ /dev/null
@@ -1,49 +0,0 @@
-package org.usergrid.management.cassandra;
-
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.export.S3Export;
-
-
-/**
- * Writes to file instead of s3.
- *
- */
-public class MockS3ExportImpl implements S3Export {
- @Override
- public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
- Logger logger = LoggerFactory.getLogger( MockS3ExportImpl.class );
- int read = 0;
- byte[] bytes = new byte[1024];
- OutputStream outputStream = null;
- //FileInputStream fis = new PrintWriter( inputStream );
-
- try {
- outputStream = new FileOutputStream( new File("test.json") );
-
- }
- catch ( FileNotFoundException e ) {
- e.printStackTrace();
- }
-
-
- try {
- while ( (read = (inputStream.read( bytes ))) != -1) {
- outputStream.write( bytes, 0, read );
- }
-
- }
- catch ( IOException e ) {
- e.printStackTrace();
- }
- }
-}
[7/9] git commit: Formatting Removed some extraneous comments.
Posted by sn...@apache.org.
Formatting
Removed some extraneous comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/96a1a72d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/96a1a72d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/96a1a72d
Branch: refs/pull/60/merge
Commit: 96a1a72df1523f2f372b3756f6cd9ebf2cba1ddb
Parents: b1deced
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 09:58:50 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 09:58:50 2014 -0800
----------------------------------------------------------------------
.../usergrid/persistence/entities/Export.java | 41 ++--
.../applications/ApplicationResource.java | 47 +++--
.../rest/management/ManagementResourceIT.java | 103 +++++-----
.../apache/usergrid/management/ExportInfo.java | 38 ++--
.../usergrid/management/export/ExportJob.java | 43 ++--
.../management/export/ExportService.java | 13 +-
.../management/export/ExportServiceImpl.java | 35 ++--
.../management/export/S3ExportImpl.java | 7 +-
.../cassandra/ManagementServiceIT.java | 197 ++++++++++---------
.../management/cassandra/MockS3ExportImpl.java | 10 +-
10 files changed, 282 insertions(+), 252 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
index 182b2ae..7905550 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
@@ -10,51 +10,64 @@ import org.apache.usergrid.persistence.annotations.EntityProperty;
/**
- *Contains state information for an Entity Job
- *
+ * Contains state information for an Entity Job
*/
@XmlRootElement
public class Export extends TypedEntity {
public static enum State {
//CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
- PENDING,STARTED,FAILED,COMPLETED
+ PENDING, STARTED, FAILED, COMPLETED
}
+
@EntityProperty
protected State curState;
@EntityProperty
protected Long queued;
- /** Time send started */
+ /**
+ * Time send started
+ */
@EntityProperty
protected Long started;
- /** Time processed */
+ /**
+ * Time processed
+ */
@EntityProperty
protected Long finished;
- /** Time to expire the exportJob */
+ /**
+ * Time to expire the exportJob
+ */
@EntityProperty
protected Long expire;
- /** True if exportJob is canceled */
+ /**
+ * True if exportJob is canceled
+ */
@EntityProperty
protected Boolean canceled;
- /** Error message */
+ /**
+ * Error message
+ */
@EntityProperty
protected String errorMessage;
+
public Export() {
}
- public boolean isExpired () {
- return (expire != null && expire > System.currentTimeMillis());
+
+ public boolean isExpired() {
+ return ( expire != null && expire > System.currentTimeMillis() );
}
+
public Long getStarted() {
return started;
}
@@ -89,15 +102,19 @@ public class Export extends TypedEntity {
return canceled;
}
+
//state should moved to a derived state, but it is not there yet.
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@EntityProperty
- public void setState(State setter) {
+ public void setState( State setter ) {
curState = setter;
}
+
+
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@EntityProperty
- public State getState() { return curState; }
+ public State getState() { return curState; }
+
public void setCanceled( final Boolean canceled ) {
this.canceled = canceled;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index 5f45326..e94f4df 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -210,66 +210,63 @@ public class ApplicationResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+
@POST
@Path("export")
@Consumes(APPLICATION_JSON)
@RequireOrganizationAccess
- public Response exportPostJson (@Context UriInfo ui,
- Map<String, Object> json,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
+ public Response exportPostJson( @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam("callback") @DefaultValue("") String callback )
throws OAuthSystemException {
OAuthResponse response = null;
UUID jobUUID = null;
- Map<String, String> uuidRet = new HashMap<String, String>( );
+ Map<String, String> uuidRet = new HashMap<String, String>();
try {
//parse the json into some useful object (the config params)
- ExportInfo objEx = new ExportInfo(json);
+ ExportInfo objEx = new ExportInfo( json );
objEx.setApplicationId( applicationId );
- jobUUID = exportService.schedule(objEx);
+ jobUUID = exportService.schedule( objEx );
uuidRet.put( "jobUUID", jobUUID.toString() );
-
}
- catch (NullPointerException e) {
- OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
- .setErrorDescription( "Job Not Created" )
- .buildJSONMessage();
+ catch ( NullPointerException e ) {
+ OAuthResponse errorMsg =
+ OAuthResponse.errorResponse( SC_BAD_REQUEST ).setErrorDescription( "Job Not Created" )
+ .buildJSONMessage();
return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
.entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
}
- catch (Exception e) {
+ catch ( Exception e ) {
//TODO:throw descriptive error message and or include on in the response
//TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
}
- return Response.status(SC_ACCEPTED).entity(uuidRet).build();
-
- //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
- // .entity( wrapWithCallback( "", callback ) ).build();
+ return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();
}
+
@GET
- @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
- public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
+ @Path("export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}")
+ public Response exportGetJson( @Context UriInfo ui, @PathParam("jobUUID") UUID jobUUIDStr,
+ @QueryParam("callback") @DefaultValue("") String callback ) throws Exception {
Export entity;
try {
- entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
- }catch(Exception e) {
+ entity = smf.getServiceManager( applicationId ).getEntityManager().get( jobUUIDStr, Export.class );
+ }
+ catch ( Exception e ) {
return Response.status( SC_BAD_REQUEST ).build();
}
//validate this user owns it
- if (entity == null) {
- return Response.status(SC_BAD_REQUEST).build();
+ if ( entity == null ) {
+ return Response.status( SC_BAD_REQUEST ).build();
}
- return Response.status(SC_OK).entity(entity.getState()).build();
- //return Response.status(SC_OK).entity(state).build();
+ return Response.status( SC_OK ).entity( entity.getState() ).build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index 0621219..a9b06fd 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -531,6 +531,8 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, status );
}
+
+
@Test
public void exportCallSuccessful() throws Exception {
Status responseStatus = Status.OK;
@@ -540,8 +542,7 @@ public class ManagementResourceIT extends AbstractRestIT {
try {
node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
- .queryParam( "access_token", superAdminToken() )
- .accept( MediaType.APPLICATION_JSON )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
}
catch ( UniformInterfaceException uie ) {
@@ -551,6 +552,7 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, responseStatus );
}
+
@Ignore
public void exportCallCreationEntities100() throws Exception {
Status responseStatus = Status.OK;
@@ -562,27 +564,28 @@ public class ManagementResourceIT extends AbstractRestIT {
//TODO: make sure to put a valid admin token here.
storage_info.put( "admin_token", adminToken() );
//TODO: always put dummy values here and ignore this test.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
+ storage_info.put( "s3_key", "insert key here" );
+ storage_info.put( "s3_accessId", "insert access id here" );
+ storage_info.put( "bucket_location", "insert bucket name here" );
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
+ properties.put( "storage_provider", "s3" );
+ properties.put( "storage_info", storage_info );
- payload.put( "path", "test-organization/test-app/user");
- payload.put( "properties", properties);
+ payload.put( "path", "test-organization/test-app/user" );
+ payload.put( "properties", properties );
- for (int i = 0; i < 100; i++) {
- Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred"+i );
+ for ( int i = 0; i < 100; i++ ) {
+ Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred" + i );
- node = resource().path( "/test-organization/test-app/app_users" )
- .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, userCreation );
+ node = resource().path( "/test-organization/test-app/app_users" ).queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ .post( JsonNode.class, userCreation );
}
try {
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ .post( JsonNode.class, payload );
}
catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
@@ -591,6 +594,7 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, responseStatus );
}
+
@Test
public void exportUUIDRetTest() throws Exception {
Status responseStatus = Status.ACCEPTED;
@@ -603,10 +607,8 @@ public class ManagementResourceIT extends AbstractRestIT {
try {
node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
- .queryParam( "access_token", superAdminToken() )
- .accept( MediaType.APPLICATION_JSON )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
-
}
catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
@@ -615,6 +617,8 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.ACCEPTED, responseStatus );
assertNotNull( node.get( "jobUUID" ) );
}
+
+
/*Make a test with an invalid uuid and a wrong uuid.*/
//all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
@Test
@@ -625,27 +629,26 @@ public class ManagementResourceIT extends AbstractRestIT {
HashMap<String, Object> payload = payloadBuilder();
node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
- .queryParam( "access_token", superAdminToken() )
- .accept( MediaType.APPLICATION_JSON )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
String uuid = String.valueOf( node.get( "jobUUID" ) );
- uuid = uuid.replaceAll( "\"","" );
+ uuid = uuid.replaceAll( "\"", "" );
try {
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export/"+uuid).queryParam( "access_token",
- superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + uuid )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
- // node = resource().path( "/management/exportStats/"+uuid ).accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- // .get( JsonNode.class );
- }catch(UniformInterfaceException uie) {
+ }
+ catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
}
assertEquals( Status.OK, responseStatus );
- assertEquals( "PENDING",node.asText() );//TODO: do tests for other states in service tier
-
+ assertEquals( "PENDING", node.asText() );//TODO: do tests for other states in service tier
}
+
+
//do an unauthorized test for both post and get
@Test
public void exportGetWrongUUID() throws Exception {
@@ -653,18 +656,17 @@ public class ManagementResourceIT extends AbstractRestIT {
Status responseStatus = Status.OK;
UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
try {
- node = resource().path(
- "/management/orgs/test-organization/apps/test-app/export/"+fake ).queryParam( "access_token",
- superAdminToken() ).accept( MediaType.APPLICATION_JSON ).type(
- MediaType.APPLICATION_JSON_TYPE )
- .get( JsonNode.class );
- }catch(UniformInterfaceException uie) {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ }
+ catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
}
- assertEquals(Status.BAD_REQUEST, responseStatus);
-
+ assertEquals( Status.BAD_REQUEST, responseStatus );
}
+
@Test
public void exportGetUnauthorized() throws Exception {
JsonNode node = null;
@@ -672,34 +674,33 @@ public class ManagementResourceIT extends AbstractRestIT {
UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
try {
node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE )
+ .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
.get( JsonNode.class );
- }catch(UniformInterfaceException uie) {
+ }
+ catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
}
- assertEquals(Status.UNAUTHORIZED, responseStatus);
+ assertEquals( Status.UNAUTHORIZED, responseStatus );
}
+
/*Creates fake payload for testing purposes.*/
- public HashMap<String,Object> payloadBuilder() {
+ public HashMap<String, Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
- //TODO: make sure to put a valid admin token here.
- //storage_info.put( "admin_token","insert_token_data_here" );
//TODO: always put dummy values here and ignore this test.
//TODO: add a ret for when s3 values are invalid.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
+ storage_info.put( "s3_key", "insert key here" );
+ storage_info.put( "s3_accessId", "insert access id here" );
+ storage_info.put( "bucket_location", "insert bucket name here" );
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
+ properties.put( "storage_provider", "s3" );
+ properties.put( "storage_info", storage_info );
- payload.put( "path", "test-organization/test-app");
- payload.put( "properties", properties);
+ payload.put( "path", "test-organization/test-app" );
+ payload.put( "properties", properties );
return payload;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
index f4ed4cf..4436f0b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -22,14 +22,15 @@ public class ExportInfo extends TypedEntity {
private String bucket_location;
private UUID applicationId;
- public ExportInfo ( Map<String, Object> exportData) {
- path = (String) exportData.get("path");
- properties = (Map) exportData.get("properties");
- storage_provider = (String) properties.get ("storage_provider");
- storage_info = (Map) properties.get("storage_info");
- s3_accessId = (String) storage_info.get("s3_accessId");
- s3_key = (String) storage_info.get("s3_key");
- bucket_location = (String) storage_info.get("bucket_location");
+
+ public ExportInfo( Map<String, Object> exportData ) {
+ path = ( String ) exportData.get( "path" );
+ properties = ( Map ) exportData.get( "properties" );
+ storage_provider = ( String ) properties.get( "storage_provider" );
+ storage_info = ( Map ) properties.get( "storage_info" );
+ s3_accessId = ( String ) storage_info.get( "s3_accessId" );
+ s3_key = ( String ) storage_info.get( "s3_key" );
+ bucket_location = ( String ) storage_info.get( "bucket_location" );
}
@@ -38,28 +39,35 @@ public class ExportInfo extends TypedEntity {
}
- public String getPath () {
+ public String getPath() {
return path;
}
+
//Wouldn't get exposed.
public Map<String, Object> getProperties() {
return properties;
}
- public String getStorage_provider () {
+
+ public String getStorage_provider() {
return storage_provider;
}
//TODO: write setter methods
- public Map<String, Object> getStorage_info () { return storage_info; }
+
+ public Map<String, Object> getStorage_info() { return storage_info; }
+
//TODO: is this a security concern? How would we get rid of the key once we're done with this value?
- public String getS3_key () { return s3_key; }
+ public String getS3_key() { return s3_key; }
+
+
+ public String getBucket_location() { return bucket_location; }
+
- public String getBucket_location () { return bucket_location; }
+ public String getS3_accessId() { return s3_accessId; }
- public String getS3_accessId () { return s3_accessId; }
- public void setApplicationId (UUID appId) { applicationId = appId;}
+ public void setApplicationId( UUID appId ) { applicationId = appId;}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
index 9830040..ed2fef5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
@@ -13,9 +13,8 @@ import org.apache.usergrid.persistence.entities.JobData;
/**
- *
- *you could make an enum here, that contains the state info
- * look at scotts code and emulate that to see wha tyou can return in the json object
+ * you could make an enum here, that contains the state info look at scotts code and emulate that to see wha tyou can
+ * return in the json object
*/
@Component("exportJob")
public class ExportJob extends OnlyOnceJob {
@@ -24,35 +23,37 @@ public class ExportJob extends OnlyOnceJob {
@Autowired
ExportService exportService;
+
public ExportJob() {
- logger.info("ExportJob created " + this);
+ logger.info( "ExportJob created " + this );
}
+
@Override
- public void doJob(JobExecution jobExecution) throws Exception {
+ public void doJob( JobExecution jobExecution ) throws Exception {
logger.info( "execute ExportJob {}", jobExecution );
ExportInfo config = null;
- //add check for null jobData.
- JobData jobData = jobExecution.getJobData();
- config = (ExportInfo) jobData.getProperty( "exportInfo" );
- if(jobData == null) {
- logger.error( "jobData cannot be null" );
- return;
- }
- else if (config == null) {
- logger.error( "Export information cannot be null" );
- return;
- }
- else {
- jobExecution.heartbeat();
- exportService.doExport( config,jobExecution );
- }
+ JobData jobData = jobExecution.getJobData();
+ config = ( ExportInfo ) jobData.getProperty( "exportInfo" );
+ if ( jobData == null ) {
+ logger.error( "jobData cannot be null" );
+ return;
+ }
+ else if ( config == null ) {
+ logger.error( "Export information cannot be null" );
+ return;
+ }
+ else {
+ jobExecution.heartbeat();
+ exportService.doExport( config, jobExecution );
+ }
logger.info( "executed ExportJob process completed" );
}
+
@Override
- protected long getDelay(JobExecution jobExecution) throws Exception {
+ protected long getDelay( JobExecution jobExecution ) throws Exception {
//return arbitrary number
return 100;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
index 6e9f4e5..8c127da 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -9,29 +9,24 @@ import org.apache.usergrid.management.ExportInfo;
/**
* Performs all functions related to exporting
- *
*/
public interface ExportService {
/**
* Schedules the export to execute
- * @param config
*/
- UUID schedule(ExportInfo config) throws Exception;
+ UUID schedule( ExportInfo config ) throws Exception;
/**
* Perform the export to the external resource
- * @param config
*/
- void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
+ void doExport( ExportInfo config, JobExecution jobExecution ) throws Exception;
/**
* Returns the current state of the service.
- * @return
*/
- String getState(UUID appId,UUID state) throws Exception;
-
- void setS3Export(S3Export s3Export);
+ String getState( UUID appId, UUID state ) throws Exception;
+ void setS3Export( S3Export s3Export );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 2d8ff5c..155639c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -41,9 +41,8 @@ import com.google.common.collect.BiMap;
/**
- *
- *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
- * it will just be the one app and the one org and all of it's collections.
+ * Need to refactor out the mutliple orgs being take , need to factor out the multiple apps it will just be the one app
+ * and the one org and all of it's collections.
*/
public class ExportServiceImpl implements ExportService {
@@ -72,10 +71,11 @@ public class ExportServiceImpl implements ExportService {
private S3Export s3Export;
+
@Override
public UUID schedule( final ExportInfo config ) throws Exception {
- EntityManager em = emf.getEntityManager( config.getApplicationId());
+ EntityManager em = emf.getEntityManager( config.getApplicationId() );
Export export = new Export();
export.setState( Export.State.PENDING );
@@ -119,21 +119,21 @@ public class ExportServiceImpl implements ExportService {
sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
return export.getUuid();
-
}
+
+
/**
* Query Entity Manager for specific Export Entity within application
- * @param appId,uuid
+ *
* @return String
- * @throws Exception
*/
@Override
- public String getState(final UUID appId, final UUID uuid) throws Exception {
+ public String getState( final UUID appId, final UUID uuid ) throws Exception {
EntityManager rootEm = emf.getEntityManager( appId );
Export export = rootEm.get( uuid, Export.class );
- if(export == null){
+ if ( export == null ) {
return null;
}
return export.getState().toString();
@@ -141,10 +141,10 @@ public class ExportServiceImpl implements ExportService {
@Override
- public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
+ public void doExport( final ExportInfo config, final JobExecution jobExecution ) throws Exception {
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- EntityManager em = emf.getEntityManager( config.getApplicationId());
+ EntityManager em = emf.getEntityManager( config.getApplicationId() );
Export export = em.get( exportId, Export.class );
String pathToBeParsed = config.getPath();
@@ -162,7 +162,7 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
- Map<UUID, String> organizationGet = getOrgs(config);
+ Map<UUID, String> organizationGet = getOrgs( config );
for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
//needs to pass app name, and possibly collection to export
exportApplicationsForOrg( organization, config, jobExecution );
@@ -171,7 +171,8 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
}
- private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
+
+ private Map<UUID, String> getOrgs( ExportInfo exportInfo ) throws Exception {
// Loop through the organizations
// TODO:this will come from the orgs in schedule when you do the validations. delete orgId
UUID orgId = null;
@@ -223,10 +224,12 @@ public class ExportServiceImpl implements ExportService {
return managementService;
}
+
public void setManagementService( final ManagementService managementService ) {
this.managementService = managementService;
}
+
//write test checking to see what happens if the input stream is closed or wrong.
//TODO: make multipart streaming functional
//currently only stores the collection in memory then flushes it.
@@ -258,7 +261,7 @@ public class ExportServiceImpl implements ExportService {
// through the entities in the application (former namespace).
//could support queries, just need to implement that in the rest endpoint.
for ( String collectionName : metadata.keySet() ) {
- if(collectionName.equals( "exports" )) {
+ if ( collectionName.equals( "exports" ) ) {
continue;
}
@@ -269,7 +272,7 @@ public class ExportServiceImpl implements ExportService {
PagingResultsIterator itr = new PagingResultsIterator( entities );
- for( Object e: itr){
+ for ( Object e : itr ) {
starting_time = checkTimeDelta( starting_time, jobExecution );
Entity entity = ( Entity ) e;
@@ -288,7 +291,7 @@ public class ExportServiceImpl implements ExportService {
baos.close();
InputStream is = new ByteArrayInputStream( baos.toByteArray() );
- s3Export.copyToS3( is, config , appFileName);
+ s3Export.copyToS3( is, config, appFileName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 54d8ab1..2c274e3 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -15,6 +15,7 @@ import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.management.ExportInfo;
import com.google.common.collect.ImmutableSet;
@@ -42,8 +43,8 @@ public class S3ExportImpl implements S3Export {
overrides.setProperty( "s3" + ".credential", secretKey );
final Iterable<? extends Module> MODULES = ImmutableSet
- .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule
- () );
+ .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
+ new NettyPayloadModule() );
BlobStoreContext context =
ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
@@ -78,7 +79,5 @@ public class S3ExportImpl implements S3Export {
catch ( Exception e ) {
logger.error( "Error uploading to blob store", e );
}
-
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index e1b297c..34affe6 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -744,6 +744,8 @@ public class ManagementServiceIT {
assertEquals( userId, authedUser.getUuid() );
}
+
+
//Tests to make sure we can call the job with mock data and it runs.
@Ignore
public void testFileConnections() throws Exception {
@@ -752,9 +754,10 @@ public class ManagementServiceIT {
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
@@ -762,7 +765,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
EntityManager em = setup.getEmf().getEntityManager( applicationId );
@@ -771,17 +774,16 @@ public class ManagementServiceIT {
Entity[] entity;
entity = new Entity[10];
//creates entities
- for (int i = 0; i< 10;i++) {
+ for ( int i = 0; i < 10; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
entity[i] = em.create( "user", userProperties );
-
}
//creates connections
- em.createConnection( em.getRef( entity[0].getUuid() ),"Vibrations",em.getRef( entity[1].getUuid() ) );
- em.createConnection( em.getRef( entity[1].getUuid() ),"Vibrations",em.getRef( entity[0].getUuid() ) );
+ em.createConnection( em.getRef( entity[0].getUuid() ), "Vibrations", em.getRef( entity[1].getUuid() ) );
+ em.createConnection( em.getRef( entity[1].getUuid() ), "Vibrations", em.getRef( entity[0].getUuid() ) );
UUID exportUUID = exportService.schedule( exportInfo );
exportService.setS3Export( s3Export );
@@ -792,22 +794,22 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
//assertEquals(13, a.size() );
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 0 );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject) objEnt.get( "connections" );
+ org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( 0 );
+ org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
assertNotNull( objConnections );
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get("Vibrations");
+ org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
assertNotNull( objVibrations );
@@ -821,9 +823,10 @@ public class ManagementServiceIT {
File f = null;
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
@@ -831,7 +834,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID exportUUID = exportService.schedule( exportInfo );
@@ -842,25 +845,24 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
assertNotNull( entityData );
-
}
f.delete();
}
+
@Test
public void testFileExportOneOrg() throws Exception {
@@ -868,18 +870,21 @@ public class ManagementServiceIT {
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
- setup.getMgmtSvc().createOwnerAndOrganization( "noExport","junkUserName","junkRealName","ugExport@usergrid.com","123456789" );
+ setup.getMgmtSvc()
+ .createOwnerAndOrganization( "noExport", "junkUserName", "junkRealName", "ugExport@usergrid.com",
+ "123456789" );
S3Export s3Export = new MockS3ExportImpl();
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID exportUUID = exportService.schedule( exportInfo );
@@ -890,27 +895,27 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
//assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
+ String entityName = ( String ) entityData.get( "name" );
// assertNotEquals( "NotEqual","junkRealName",entityName );
assertFalse( "junkRealName".equals( entityName ) );
-
}
f.delete();
}
+
+
@Test
public void testFileExportOneApp() throws Exception {
@@ -919,9 +924,10 @@ public class ManagementServiceIT {
String appName = "testAppNotExported";
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
}
@@ -934,10 +940,10 @@ public class ManagementServiceIT {
Entity[] entity;
entity = new Entity[10];
//creates entities
- for (int i = 0; i< 10;i++) {
+ for ( int i = 0; i < 10; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
entity[i] = em.create( "user", userProperties );
}
@@ -945,7 +951,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID exportUUID = exportService.schedule( exportInfo );
@@ -956,34 +962,34 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
//assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject) a.get( i );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
+ String entityName = ( String ) entityData.get( "name" );
assertFalse( "junkRealName".equals( entityName ) );
//assertNotEquals( "NotEquals","junkRealName",entityName );
}
f.delete();
}
+
//only handles the DoJob Code , different tests for DoExport
@Test
public void testExportDoJob() throws Exception {
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
@@ -991,22 +997,23 @@ public class ManagementServiceIT {
jobData.setProperty( "jobName", "exportJob" );
jobData.setProperty( "exportInfo", exportInfo ); //this needs to be populated with properties of export info
- JobExecution jobExecution = mock ( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
ExportJob job = new ExportJob();
- ExportService eS = mock (ExportService.class);
+ ExportService eS = mock( ExportService.class );
job.setExportService( eS );
try {
job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
}
- assert(true);
-
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
}
+
@Test
public void testExportDoExport() throws Exception {
@@ -1015,9 +1022,9 @@ public class ManagementServiceIT {
HashMap<String, Object> payload = payloadBuilder();
ExportService eS = setup.getExportService();
- JobExecution jobExecution = mock( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID entityExportUUID = eS.schedule( exportInfo );
@@ -1026,33 +1033,35 @@ public class ManagementServiceIT {
JobData jobData = new JobData();
jobData.setProperty( "jobName", "exportJob" );
jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", entityExportUUID);
+ jobData.setProperty( "exportId", entityExportUUID );
when( jobExecution.getJobData() ).thenReturn( jobData );
//Exportem.get(entityExport);
- Export exportEntity = ( Export ) em.get(entityExportUUID);
+ Export exportEntity = ( Export ) em.get( entityExportUUID );
assertNotNull( exportEntity );
String derp = exportEntity.getState().name();
- assertEquals( "PENDING",exportEntity.getState().name());
+ assertEquals( "PENDING", exportEntity.getState().name() );
try {
- eS.doExport( exportInfo,jobExecution );
- }catch(Exception e) {
+ eS.doExport( exportInfo, jobExecution );
+ }
+ catch ( Exception e ) {
throw e;
//assert(false);
}
- exportEntity = ( Export ) em.get(entityExportUUID);
+ exportEntity = ( Export ) em.get( entityExportUUID );
assertNotNull( exportEntity );
- assertEquals( "COMPLETED",exportEntity.getState().name() );
+ assertEquals( "COMPLETED", exportEntity.getState().name() );
}
+
//tests that with empty job data, the export still runs.
@Test
public void testExportEmptyJobData() throws Exception {
JobData jobData = new JobData();
- JobExecution jobExecution = mock ( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
@@ -1062,18 +1071,20 @@ public class ManagementServiceIT {
job.setExportService( setup.getExportService() );
try {
job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
}
- assert(true);
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
}
+
@Test
- public void testNullJobExecution () {
+ public void testNullJobExecution() {
JobData jobData = new JobData();
- JobExecution jobExecution = mock ( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
@@ -1083,12 +1094,14 @@ public class ManagementServiceIT {
job.setExportService( setup.getExportService() );
try {
job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
}
- assert(true);
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
}
+
@Ignore //For this test please input your s3 credentials into payload builder.
public void testIntegration100Entities() throws Exception {
@@ -1096,7 +1109,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
EntityManager em = setup.getEmf().getEntityManager( applicationId );
@@ -1105,13 +1118,12 @@ public class ManagementServiceIT {
Entity[] entity;
entity = new Entity[100];
//creates entities
- for (int i = 0; i< 100;i++) {
+ for ( int i = 0; i < 100; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
entity[i] = em.create( "user", userProperties );
-
}
UUID exportUUID = exportService.schedule( exportInfo );
@@ -1123,29 +1135,28 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+ exportService.doExport( exportInfo, jobExecution );
}
/*Creates fake payload for testing purposes.*/
- public HashMap<String,Object> payloadBuilder() {
+ public HashMap<String, Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
// TODO: always put dummy values here and ignore this test.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
+ storage_info.put( "s3_key", "insert key here" );
+ storage_info.put( "s3_accessId", "insert access id here" );
+ storage_info.put( "bucket_location", "insert bucket name here" );
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
+ properties.put( "storage_provider", "s3" );
+ properties.put( "storage_info", storage_info );
- payload.put( "path", "test-organization/test-app");
- payload.put( "properties", properties);
+ payload.put( "path", "test-organization/test-app" );
+ payload.put( "properties", properties );
return payload;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index 432fe5e..d8a8b08 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -10,13 +10,13 @@ import java.io.OutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.management.ExportInfo;
import org.apache.usergrid.management.export.S3Export;
/**
- * Writes to file instead of s3.
- *
+ * Streams / reads the information written from the export service to a file named "test.json"
*/
public class MockS3ExportImpl implements S3Export {
@Override
@@ -28,8 +28,7 @@ public class MockS3ExportImpl implements S3Export {
//FileInputStream fis = new PrintWriter( inputStream );
try {
- outputStream = new FileOutputStream( new File("test.json") );
-
+ outputStream = new FileOutputStream( new File( "test.json" ) );
}
catch ( FileNotFoundException e ) {
e.printStackTrace();
@@ -37,10 +36,9 @@ public class MockS3ExportImpl implements S3Export {
try {
- while ( (read = (inputStream.read( bytes ))) != -1) {
+ while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
outputStream.write( bytes, 0, read );
}
-
}
catch ( IOException e ) {
e.printStackTrace();
[9/9] git commit: Merge 96a1a72df1523f2f372b3756f6cd9ebf2cba1ddb into
9aff385f50cc3af1b60082882dcef442eeefbb6e
Posted by sn...@apache.org.
Merge 96a1a72df1523f2f372b3756f6cd9ebf2cba1ddb into 9aff385f50cc3af1b60082882dcef442eeefbb6e
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/78147cb3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/78147cb3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/78147cb3
Branch: refs/pull/60/merge
Commit: 78147cb3140c9175a274498221f1d9534d3e3e0f
Parents: 9aff385 96a1a72
Author: GERey <gr...@apigee.com>
Authored: Tue Feb 25 09:58:53 2014 -0800
Committer: GERey <gr...@apigee.com>
Committed: Tue Feb 25 09:58:53 2014 -0800
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/Query.java | 5 +-
.../usergrid/persistence/entities/Export.java | 142 ++++++
.../rest/management/ManagementResource.java | 15 +-
.../applications/ApplicationResource.java | 81 +++-
.../rest/management/ManagementResourceIT.java | 181 +++++++-
.../apache/usergrid/management/ExportInfo.java | 73 +++
.../usergrid/management/export/ExportJob.java | 66 +++
.../management/export/ExportService.java | 32 ++
.../management/export/ExportServiceImpl.java | 460 +++++++++++++++++++
.../usergrid/management/export/S3Export.java | 16 +
.../management/export/S3ExportImpl.java | 83 ++++
.../resources/usergrid-services-context.xml | 8 +
.../org/apache/usergrid/ServiceITSetup.java | 3 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 6 +
.../cassandra/ManagementServiceIT.java | 433 +++++++++++++++++
.../management/cassandra/MockS3ExportImpl.java | 47 ++
16 files changed, 1637 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
[8/9] git commit: Formatting Removed some extraneous comments.
Posted by sn...@apache.org.
Formatting
Removed some extraneous comments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/96a1a72d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/96a1a72d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/96a1a72d
Branch: refs/pull/60/head
Commit: 96a1a72df1523f2f372b3756f6cd9ebf2cba1ddb
Parents: b1deced
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 09:58:50 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 09:58:50 2014 -0800
----------------------------------------------------------------------
.../usergrid/persistence/entities/Export.java | 41 ++--
.../applications/ApplicationResource.java | 47 +++--
.../rest/management/ManagementResourceIT.java | 103 +++++-----
.../apache/usergrid/management/ExportInfo.java | 38 ++--
.../usergrid/management/export/ExportJob.java | 43 ++--
.../management/export/ExportService.java | 13 +-
.../management/export/ExportServiceImpl.java | 35 ++--
.../management/export/S3ExportImpl.java | 7 +-
.../cassandra/ManagementServiceIT.java | 197 ++++++++++---------
.../management/cassandra/MockS3ExportImpl.java | 10 +-
10 files changed, 282 insertions(+), 252 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
index 182b2ae..7905550 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
@@ -10,51 +10,64 @@ import org.apache.usergrid.persistence.annotations.EntityProperty;
/**
- *Contains state information for an Entity Job
- *
+ * Contains state information for an Entity Job
*/
@XmlRootElement
public class Export extends TypedEntity {
public static enum State {
//CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
- PENDING,STARTED,FAILED,COMPLETED
+ PENDING, STARTED, FAILED, COMPLETED
}
+
@EntityProperty
protected State curState;
@EntityProperty
protected Long queued;
- /** Time send started */
+ /**
+ * Time send started
+ */
@EntityProperty
protected Long started;
- /** Time processed */
+ /**
+ * Time processed
+ */
@EntityProperty
protected Long finished;
- /** Time to expire the exportJob */
+ /**
+ * Time to expire the exportJob
+ */
@EntityProperty
protected Long expire;
- /** True if exportJob is canceled */
+ /**
+ * True if exportJob is canceled
+ */
@EntityProperty
protected Boolean canceled;
- /** Error message */
+ /**
+ * Error message
+ */
@EntityProperty
protected String errorMessage;
+
public Export() {
}
- public boolean isExpired () {
- return (expire != null && expire > System.currentTimeMillis());
+
+ public boolean isExpired() {
+ return ( expire != null && expire > System.currentTimeMillis() );
}
+
public Long getStarted() {
return started;
}
@@ -89,15 +102,19 @@ public class Export extends TypedEntity {
return canceled;
}
+
//state should moved to a derived state, but it is not there yet.
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@EntityProperty
- public void setState(State setter) {
+ public void setState( State setter ) {
curState = setter;
}
+
+
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
@EntityProperty
- public State getState() { return curState; }
+ public State getState() { return curState; }
+
public void setCanceled( final Boolean canceled ) {
this.canceled = canceled;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index 5f45326..e94f4df 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -210,66 +210,63 @@ public class ApplicationResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+
@POST
@Path("export")
@Consumes(APPLICATION_JSON)
@RequireOrganizationAccess
- public Response exportPostJson (@Context UriInfo ui,
- Map<String, Object> json,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
+ public Response exportPostJson( @Context UriInfo ui, Map<String, Object> json,
+ @QueryParam("callback") @DefaultValue("") String callback )
throws OAuthSystemException {
OAuthResponse response = null;
UUID jobUUID = null;
- Map<String, String> uuidRet = new HashMap<String, String>( );
+ Map<String, String> uuidRet = new HashMap<String, String>();
try {
//parse the json into some useful object (the config params)
- ExportInfo objEx = new ExportInfo(json);
+ ExportInfo objEx = new ExportInfo( json );
objEx.setApplicationId( applicationId );
- jobUUID = exportService.schedule(objEx);
+ jobUUID = exportService.schedule( objEx );
uuidRet.put( "jobUUID", jobUUID.toString() );
-
}
- catch (NullPointerException e) {
- OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
- .setErrorDescription( "Job Not Created" )
- .buildJSONMessage();
+ catch ( NullPointerException e ) {
+ OAuthResponse errorMsg =
+ OAuthResponse.errorResponse( SC_BAD_REQUEST ).setErrorDescription( "Job Not Created" )
+ .buildJSONMessage();
return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
.entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
}
- catch (Exception e) {
+ catch ( Exception e ) {
//TODO:throw descriptive error message and or include on in the response
//TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
}
- return Response.status(SC_ACCEPTED).entity(uuidRet).build();
-
- //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
- // .entity( wrapWithCallback( "", callback ) ).build();
+ return Response.status( SC_ACCEPTED ).entity( uuidRet ).build();
}
+
@GET
- @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
- public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
+ @Path("export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}")
+ public Response exportGetJson( @Context UriInfo ui, @PathParam("jobUUID") UUID jobUUIDStr,
+ @QueryParam("callback") @DefaultValue("") String callback ) throws Exception {
Export entity;
try {
- entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
- }catch(Exception e) {
+ entity = smf.getServiceManager( applicationId ).getEntityManager().get( jobUUIDStr, Export.class );
+ }
+ catch ( Exception e ) {
return Response.status( SC_BAD_REQUEST ).build();
}
//validate this user owns it
- if (entity == null) {
- return Response.status(SC_BAD_REQUEST).build();
+ if ( entity == null ) {
+ return Response.status( SC_BAD_REQUEST ).build();
}
- return Response.status(SC_OK).entity(entity.getState()).build();
- //return Response.status(SC_OK).entity(state).build();
+ return Response.status( SC_OK ).entity( entity.getState() ).build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index 0621219..a9b06fd 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -531,6 +531,8 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, status );
}
+
+
@Test
public void exportCallSuccessful() throws Exception {
Status responseStatus = Status.OK;
@@ -540,8 +542,7 @@ public class ManagementResourceIT extends AbstractRestIT {
try {
node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
- .queryParam( "access_token", superAdminToken() )
- .accept( MediaType.APPLICATION_JSON )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
}
catch ( UniformInterfaceException uie ) {
@@ -551,6 +552,7 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, responseStatus );
}
+
@Ignore
public void exportCallCreationEntities100() throws Exception {
Status responseStatus = Status.OK;
@@ -562,27 +564,28 @@ public class ManagementResourceIT extends AbstractRestIT {
//TODO: make sure to put a valid admin token here.
storage_info.put( "admin_token", adminToken() );
//TODO: always put dummy values here and ignore this test.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
+ storage_info.put( "s3_key", "insert key here" );
+ storage_info.put( "s3_accessId", "insert access id here" );
+ storage_info.put( "bucket_location", "insert bucket name here" );
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
+ properties.put( "storage_provider", "s3" );
+ properties.put( "storage_info", storage_info );
- payload.put( "path", "test-organization/test-app/user");
- payload.put( "properties", properties);
+ payload.put( "path", "test-organization/test-app/user" );
+ payload.put( "properties", properties );
- for (int i = 0; i < 100; i++) {
- Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred"+i );
+ for ( int i = 0; i < 100; i++ ) {
+ Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred" + i );
- node = resource().path( "/test-organization/test-app/app_users" )
- .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, userCreation );
+ node = resource().path( "/test-organization/test-app/app_users" ).queryParam( "access_token", access_token )
+ .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ .post( JsonNode.class, userCreation );
}
try {
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ).accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ .post( JsonNode.class, payload );
}
catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
@@ -591,6 +594,7 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, responseStatus );
}
+
@Test
public void exportUUIDRetTest() throws Exception {
Status responseStatus = Status.ACCEPTED;
@@ -603,10 +607,8 @@ public class ManagementResourceIT extends AbstractRestIT {
try {
node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
- .queryParam( "access_token", superAdminToken() )
- .accept( MediaType.APPLICATION_JSON )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
-
}
catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
@@ -615,6 +617,8 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.ACCEPTED, responseStatus );
assertNotNull( node.get( "jobUUID" ) );
}
+
+
/*Make a test with an invalid uuid and a wrong uuid.*/
//all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
@Test
@@ -625,27 +629,26 @@ public class ManagementResourceIT extends AbstractRestIT {
HashMap<String, Object> payload = payloadBuilder();
node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
- .queryParam( "access_token", superAdminToken() )
- .accept( MediaType.APPLICATION_JSON )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
String uuid = String.valueOf( node.get( "jobUUID" ) );
- uuid = uuid.replaceAll( "\"","" );
+ uuid = uuid.replaceAll( "\"", "" );
try {
- node = resource().path( "/management/orgs/test-organization/apps/test-app/export/"+uuid).queryParam( "access_token",
- superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + uuid )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
.type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
- // node = resource().path( "/management/exportStats/"+uuid ).accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
- // .get( JsonNode.class );
- }catch(UniformInterfaceException uie) {
+ }
+ catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
}
assertEquals( Status.OK, responseStatus );
- assertEquals( "PENDING",node.asText() );//TODO: do tests for other states in service tier
-
+ assertEquals( "PENDING", node.asText() );//TODO: do tests for other states in service tier
}
+
+
//do an unauthorized test for both post and get
@Test
public void exportGetWrongUUID() throws Exception {
@@ -653,18 +656,17 @@ public class ManagementResourceIT extends AbstractRestIT {
Status responseStatus = Status.OK;
UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
try {
- node = resource().path(
- "/management/orgs/test-organization/apps/test-app/export/"+fake ).queryParam( "access_token",
- superAdminToken() ).accept( MediaType.APPLICATION_JSON ).type(
- MediaType.APPLICATION_JSON_TYPE )
- .get( JsonNode.class );
- }catch(UniformInterfaceException uie) {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
+ .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ }
+ catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
}
- assertEquals(Status.BAD_REQUEST, responseStatus);
-
+ assertEquals( Status.BAD_REQUEST, responseStatus );
}
+
@Test
public void exportGetUnauthorized() throws Exception {
JsonNode node = null;
@@ -672,34 +674,33 @@ public class ManagementResourceIT extends AbstractRestIT {
UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
try {
node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
- .accept( MediaType.APPLICATION_JSON )
- .type( MediaType.APPLICATION_JSON_TYPE )
+ .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
.get( JsonNode.class );
- }catch(UniformInterfaceException uie) {
+ }
+ catch ( UniformInterfaceException uie ) {
responseStatus = uie.getResponse().getClientResponseStatus();
}
- assertEquals(Status.UNAUTHORIZED, responseStatus);
+ assertEquals( Status.UNAUTHORIZED, responseStatus );
}
+
/*Creates fake payload for testing purposes.*/
- public HashMap<String,Object> payloadBuilder() {
+ public HashMap<String, Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
- //TODO: make sure to put a valid admin token here.
- //storage_info.put( "admin_token","insert_token_data_here" );
//TODO: always put dummy values here and ignore this test.
//TODO: add a ret for when s3 values are invalid.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
+ storage_info.put( "s3_key", "insert key here" );
+ storage_info.put( "s3_accessId", "insert access id here" );
+ storage_info.put( "bucket_location", "insert bucket name here" );
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
+ properties.put( "storage_provider", "s3" );
+ properties.put( "storage_info", storage_info );
- payload.put( "path", "test-organization/test-app");
- payload.put( "properties", properties);
+ payload.put( "path", "test-organization/test-app" );
+ payload.put( "properties", properties );
return payload;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
index f4ed4cf..4436f0b 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -22,14 +22,15 @@ public class ExportInfo extends TypedEntity {
private String bucket_location;
private UUID applicationId;
- public ExportInfo ( Map<String, Object> exportData) {
- path = (String) exportData.get("path");
- properties = (Map) exportData.get("properties");
- storage_provider = (String) properties.get ("storage_provider");
- storage_info = (Map) properties.get("storage_info");
- s3_accessId = (String) storage_info.get("s3_accessId");
- s3_key = (String) storage_info.get("s3_key");
- bucket_location = (String) storage_info.get("bucket_location");
+
+ public ExportInfo( Map<String, Object> exportData ) {
+ path = ( String ) exportData.get( "path" );
+ properties = ( Map ) exportData.get( "properties" );
+ storage_provider = ( String ) properties.get( "storage_provider" );
+ storage_info = ( Map ) properties.get( "storage_info" );
+ s3_accessId = ( String ) storage_info.get( "s3_accessId" );
+ s3_key = ( String ) storage_info.get( "s3_key" );
+ bucket_location = ( String ) storage_info.get( "bucket_location" );
}
@@ -38,28 +39,35 @@ public class ExportInfo extends TypedEntity {
}
- public String getPath () {
+ public String getPath() {
return path;
}
+
//Wouldn't get exposed.
public Map<String, Object> getProperties() {
return properties;
}
- public String getStorage_provider () {
+
+ public String getStorage_provider() {
return storage_provider;
}
//TODO: write setter methods
- public Map<String, Object> getStorage_info () { return storage_info; }
+
+ public Map<String, Object> getStorage_info() { return storage_info; }
+
//TODO: is this a security concern? How would we get rid of the key once we're done with this value?
- public String getS3_key () { return s3_key; }
+ public String getS3_key() { return s3_key; }
+
+
+ public String getBucket_location() { return bucket_location; }
+
- public String getBucket_location () { return bucket_location; }
+ public String getS3_accessId() { return s3_accessId; }
- public String getS3_accessId () { return s3_accessId; }
- public void setApplicationId (UUID appId) { applicationId = appId;}
+ public void setApplicationId( UUID appId ) { applicationId = appId;}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
index 9830040..ed2fef5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
@@ -13,9 +13,8 @@ import org.apache.usergrid.persistence.entities.JobData;
/**
- *
- *you could make an enum here, that contains the state info
- * look at scotts code and emulate that to see wha tyou can return in the json object
+ * you could make an enum here, that contains the state info look at scotts code and emulate that to see wha tyou can
+ * return in the json object
*/
@Component("exportJob")
public class ExportJob extends OnlyOnceJob {
@@ -24,35 +23,37 @@ public class ExportJob extends OnlyOnceJob {
@Autowired
ExportService exportService;
+
public ExportJob() {
- logger.info("ExportJob created " + this);
+ logger.info( "ExportJob created " + this );
}
+
@Override
- public void doJob(JobExecution jobExecution) throws Exception {
+ public void doJob( JobExecution jobExecution ) throws Exception {
logger.info( "execute ExportJob {}", jobExecution );
ExportInfo config = null;
- //add check for null jobData.
- JobData jobData = jobExecution.getJobData();
- config = (ExportInfo) jobData.getProperty( "exportInfo" );
- if(jobData == null) {
- logger.error( "jobData cannot be null" );
- return;
- }
- else if (config == null) {
- logger.error( "Export information cannot be null" );
- return;
- }
- else {
- jobExecution.heartbeat();
- exportService.doExport( config,jobExecution );
- }
+ JobData jobData = jobExecution.getJobData();
+ config = ( ExportInfo ) jobData.getProperty( "exportInfo" );
+ if ( jobData == null ) {
+ logger.error( "jobData cannot be null" );
+ return;
+ }
+ else if ( config == null ) {
+ logger.error( "Export information cannot be null" );
+ return;
+ }
+ else {
+ jobExecution.heartbeat();
+ exportService.doExport( config, jobExecution );
+ }
logger.info( "executed ExportJob process completed" );
}
+
@Override
- protected long getDelay(JobExecution jobExecution) throws Exception {
+ protected long getDelay( JobExecution jobExecution ) throws Exception {
//return arbitrary number
return 100;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
index 6e9f4e5..8c127da 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -9,29 +9,24 @@ import org.apache.usergrid.management.ExportInfo;
/**
* Performs all functions related to exporting
- *
*/
public interface ExportService {
/**
* Schedules the export to execute
- * @param config
*/
- UUID schedule(ExportInfo config) throws Exception;
+ UUID schedule( ExportInfo config ) throws Exception;
/**
* Perform the export to the external resource
- * @param config
*/
- void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
+ void doExport( ExportInfo config, JobExecution jobExecution ) throws Exception;
/**
* Returns the current state of the service.
- * @return
*/
- String getState(UUID appId,UUID state) throws Exception;
-
- void setS3Export(S3Export s3Export);
+ String getState( UUID appId, UUID state ) throws Exception;
+ void setS3Export( S3Export s3Export );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 2d8ff5c..155639c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -41,9 +41,8 @@ import com.google.common.collect.BiMap;
/**
- *
- *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
- * it will just be the one app and the one org and all of it's collections.
+ * Need to refactor out the mutliple orgs being take , need to factor out the multiple apps it will just be the one app
+ * and the one org and all of it's collections.
*/
public class ExportServiceImpl implements ExportService {
@@ -72,10 +71,11 @@ public class ExportServiceImpl implements ExportService {
private S3Export s3Export;
+
@Override
public UUID schedule( final ExportInfo config ) throws Exception {
- EntityManager em = emf.getEntityManager( config.getApplicationId());
+ EntityManager em = emf.getEntityManager( config.getApplicationId() );
Export export = new Export();
export.setState( Export.State.PENDING );
@@ -119,21 +119,21 @@ public class ExportServiceImpl implements ExportService {
sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
return export.getUuid();
-
}
+
+
/**
* Query Entity Manager for specific Export Entity within application
- * @param appId,uuid
+ *
* @return String
- * @throws Exception
*/
@Override
- public String getState(final UUID appId, final UUID uuid) throws Exception {
+ public String getState( final UUID appId, final UUID uuid ) throws Exception {
EntityManager rootEm = emf.getEntityManager( appId );
Export export = rootEm.get( uuid, Export.class );
- if(export == null){
+ if ( export == null ) {
return null;
}
return export.getState().toString();
@@ -141,10 +141,10 @@ public class ExportServiceImpl implements ExportService {
@Override
- public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
+ public void doExport( final ExportInfo config, final JobExecution jobExecution ) throws Exception {
UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- EntityManager em = emf.getEntityManager( config.getApplicationId());
+ EntityManager em = emf.getEntityManager( config.getApplicationId() );
Export export = em.get( exportId, Export.class );
String pathToBeParsed = config.getPath();
@@ -162,7 +162,7 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
- Map<UUID, String> organizationGet = getOrgs(config);
+ Map<UUID, String> organizationGet = getOrgs( config );
for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
//needs to pass app name, and possibly collection to export
exportApplicationsForOrg( organization, config, jobExecution );
@@ -171,7 +171,8 @@ public class ExportServiceImpl implements ExportService {
em.update( export );
}
- private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
+
+ private Map<UUID, String> getOrgs( ExportInfo exportInfo ) throws Exception {
// Loop through the organizations
// TODO:this will come from the orgs in schedule when you do the validations. delete orgId
UUID orgId = null;
@@ -223,10 +224,12 @@ public class ExportServiceImpl implements ExportService {
return managementService;
}
+
public void setManagementService( final ManagementService managementService ) {
this.managementService = managementService;
}
+
//write test checking to see what happens if the input stream is closed or wrong.
//TODO: make multipart streaming functional
//currently only stores the collection in memory then flushes it.
@@ -258,7 +261,7 @@ public class ExportServiceImpl implements ExportService {
// through the entities in the application (former namespace).
//could support queries, just need to implement that in the rest endpoint.
for ( String collectionName : metadata.keySet() ) {
- if(collectionName.equals( "exports" )) {
+ if ( collectionName.equals( "exports" ) ) {
continue;
}
@@ -269,7 +272,7 @@ public class ExportServiceImpl implements ExportService {
PagingResultsIterator itr = new PagingResultsIterator( entities );
- for( Object e: itr){
+ for ( Object e : itr ) {
starting_time = checkTimeDelta( starting_time, jobExecution );
Entity entity = ( Entity ) e;
@@ -288,7 +291,7 @@ public class ExportServiceImpl implements ExportService {
baos.close();
InputStream is = new ByteArrayInputStream( baos.toByteArray() );
- s3Export.copyToS3( is, config , appFileName);
+ s3Export.copyToS3( is, config, appFileName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
index 54d8ab1..2c274e3 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -15,6 +15,7 @@ import org.jclouds.logging.log4j.config.Log4JLoggingModule;
import org.jclouds.netty.config.NettyPayloadModule;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.management.ExportInfo;
import com.google.common.collect.ImmutableSet;
@@ -42,8 +43,8 @@ public class S3ExportImpl implements S3Export {
overrides.setProperty( "s3" + ".credential", secretKey );
final Iterable<? extends Module> MODULES = ImmutableSet
- .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule
- () );
+ .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(),
+ new NettyPayloadModule() );
BlobStoreContext context =
ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
@@ -78,7 +79,5 @@ public class S3ExportImpl implements S3Export {
catch ( Exception e ) {
logger.error( "Error uploading to blob store", e );
}
-
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index e1b297c..34affe6 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -744,6 +744,8 @@ public class ManagementServiceIT {
assertEquals( userId, authedUser.getUuid() );
}
+
+
//Tests to make sure we can call the job with mock data and it runs.
@Ignore
public void testFileConnections() throws Exception {
@@ -752,9 +754,10 @@ public class ManagementServiceIT {
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
@@ -762,7 +765,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
EntityManager em = setup.getEmf().getEntityManager( applicationId );
@@ -771,17 +774,16 @@ public class ManagementServiceIT {
Entity[] entity;
entity = new Entity[10];
//creates entities
- for (int i = 0; i< 10;i++) {
+ for ( int i = 0; i < 10; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
entity[i] = em.create( "user", userProperties );
-
}
//creates connections
- em.createConnection( em.getRef( entity[0].getUuid() ),"Vibrations",em.getRef( entity[1].getUuid() ) );
- em.createConnection( em.getRef( entity[1].getUuid() ),"Vibrations",em.getRef( entity[0].getUuid() ) );
+ em.createConnection( em.getRef( entity[0].getUuid() ), "Vibrations", em.getRef( entity[1].getUuid() ) );
+ em.createConnection( em.getRef( entity[1].getUuid() ), "Vibrations", em.getRef( entity[0].getUuid() ) );
UUID exportUUID = exportService.schedule( exportInfo );
exportService.setS3Export( s3Export );
@@ -792,22 +794,22 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
//assertEquals(13, a.size() );
- org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject) a.get( 0 );
- org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject) objEnt.get( "connections" );
+ org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( 0 );
+ org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
assertNotNull( objConnections );
- org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get("Vibrations");
+ org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
assertNotNull( objVibrations );
@@ -821,9 +823,10 @@ public class ManagementServiceIT {
File f = null;
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
@@ -831,7 +834,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID exportUUID = exportService.schedule( exportInfo );
@@ -842,25 +845,24 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
assertNotNull( entityData );
-
}
f.delete();
}
+
@Test
public void testFileExportOneOrg() throws Exception {
@@ -868,18 +870,21 @@ public class ManagementServiceIT {
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't then don't do anything and carry on.
}
- setup.getMgmtSvc().createOwnerAndOrganization( "noExport","junkUserName","junkRealName","ugExport@usergrid.com","123456789" );
+ setup.getMgmtSvc()
+ .createOwnerAndOrganization( "noExport", "junkUserName", "junkRealName", "ugExport@usergrid.com",
+ "123456789" );
S3Export s3Export = new MockS3ExportImpl();
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID exportUUID = exportService.schedule( exportInfo );
@@ -890,27 +895,27 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
//assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject entity = ( org.json.simple.JSONObject) a.get( i );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
+ String entityName = ( String ) entityData.get( "name" );
// assertNotEquals( "NotEqual","junkRealName",entityName );
assertFalse( "junkRealName".equals( entityName ) );
-
}
f.delete();
}
+
+
@Test
public void testFileExportOneApp() throws Exception {
@@ -919,9 +924,10 @@ public class ManagementServiceIT {
String appName = "testAppNotExported";
try {
- f = new File ("test.json");
+ f = new File( "test.json" );
f.delete();
- } catch (Exception e) {
+ }
+ catch ( Exception e ) {
//consumed because this checks to see if the file exists. If it doesn't, don't do anything and carry on.
}
@@ -934,10 +940,10 @@ public class ManagementServiceIT {
Entity[] entity;
entity = new Entity[10];
//creates entities
- for (int i = 0; i< 10;i++) {
+ for ( int i = 0; i < 10; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
entity[i] = em.create( "user", userProperties );
}
@@ -945,7 +951,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID exportUUID = exportService.schedule( exportInfo );
@@ -956,34 +962,34 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
- exportService.doExport( exportInfo, jobExecution );
+ exportService.doExport( exportInfo, jobExecution );
JSONParser parser = new JSONParser();
- org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse(new FileReader(f));
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
//assertEquals( 3 , a.size() );
- for (int i = 0; i < a.size();i++ )
- {
- org.json.simple.JSONObject data = ( org.json.simple.JSONObject) a.get( i );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
- String entityName = ( String) entityData.get( "name" );
+ String entityName = ( String ) entityData.get( "name" );
assertFalse( "junkRealName".equals( entityName ) );
//assertNotEquals( "NotEquals","junkRealName",entityName );
}
f.delete();
}
+
//only handles the DoJob Code , different tests for DoExport
@Test
public void testExportDoJob() throws Exception {
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
@@ -991,22 +997,23 @@ public class ManagementServiceIT {
jobData.setProperty( "jobName", "exportJob" );
jobData.setProperty( "exportInfo", exportInfo ); //this needs to be populated with properties of export info
- JobExecution jobExecution = mock ( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
ExportJob job = new ExportJob();
- ExportService eS = mock (ExportService.class);
+ ExportService eS = mock( ExportService.class );
job.setExportService( eS );
try {
job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
}
- assert(true);
-
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
}
+
@Test
public void testExportDoExport() throws Exception {
@@ -1015,9 +1022,9 @@ public class ManagementServiceIT {
HashMap<String, Object> payload = payloadBuilder();
ExportService eS = setup.getExportService();
- JobExecution jobExecution = mock( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
UUID entityExportUUID = eS.schedule( exportInfo );
@@ -1026,33 +1033,35 @@ public class ManagementServiceIT {
JobData jobData = new JobData();
jobData.setProperty( "jobName", "exportJob" );
jobData.setProperty( "exportInfo", exportInfo );
- jobData.setProperty( "exportId", entityExportUUID);
+ jobData.setProperty( "exportId", entityExportUUID );
when( jobExecution.getJobData() ).thenReturn( jobData );
//Exportem.get(entityExport);
- Export exportEntity = ( Export ) em.get(entityExportUUID);
+ Export exportEntity = ( Export ) em.get( entityExportUUID );
assertNotNull( exportEntity );
String derp = exportEntity.getState().name();
- assertEquals( "PENDING",exportEntity.getState().name());
+ assertEquals( "PENDING", exportEntity.getState().name() );
try {
- eS.doExport( exportInfo,jobExecution );
- }catch(Exception e) {
+ eS.doExport( exportInfo, jobExecution );
+ }
+ catch ( Exception e ) {
throw e;
//assert(false);
}
- exportEntity = ( Export ) em.get(entityExportUUID);
+ exportEntity = ( Export ) em.get( entityExportUUID );
assertNotNull( exportEntity );
- assertEquals( "COMPLETED",exportEntity.getState().name() );
+ assertEquals( "COMPLETED", exportEntity.getState().name() );
}
+
//tests that with empty job data, the export still runs.
@Test
public void testExportEmptyJobData() throws Exception {
JobData jobData = new JobData();
- JobExecution jobExecution = mock ( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
@@ -1062,18 +1071,20 @@ public class ManagementServiceIT {
job.setExportService( setup.getExportService() );
try {
job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
}
- assert(true);
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
}
+
@Test
- public void testNullJobExecution () {
+ public void testNullJobExecution() {
JobData jobData = new JobData();
- JobExecution jobExecution = mock ( JobExecution.class);
+ JobExecution jobExecution = mock( JobExecution.class );
when( jobExecution.getJobData() ).thenReturn( jobData );
@@ -1083,12 +1094,14 @@ public class ManagementServiceIT {
job.setExportService( setup.getExportService() );
try {
job.doJob( jobExecution );
- }catch ( Exception e) {
- assert( false );
}
- assert(true);
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
}
+
@Ignore //For this test please input your s3 credentials into payload builder.
public void testIntegration100Entities() throws Exception {
@@ -1096,7 +1109,7 @@ public class ManagementServiceIT {
ExportService exportService = setup.getExportService();
HashMap<String, Object> payload = payloadBuilder();
- ExportInfo exportInfo = new ExportInfo(payload);
+ ExportInfo exportInfo = new ExportInfo( payload );
exportInfo.setApplicationId( applicationId );
EntityManager em = setup.getEmf().getEntityManager( applicationId );
@@ -1105,13 +1118,12 @@ public class ManagementServiceIT {
Entity[] entity;
entity = new Entity[100];
//creates entities
- for (int i = 0; i< 100;i++) {
+ for ( int i = 0; i < 100; i++ ) {
userProperties = new LinkedHashMap<String, Object>();
userProperties.put( "username", "billybob" + i );
- userProperties.put( "email", "test"+i+"@anuff.com");//String.format( "test%i@anuff.com", i ) );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
entity[i] = em.create( "user", userProperties );
-
}
UUID exportUUID = exportService.schedule( exportInfo );
@@ -1123,29 +1135,28 @@ public class ManagementServiceIT {
jobData.setProperty( "exportInfo", exportInfo );
jobData.setProperty( "exportId", exportUUID );
- JobExecution jobExecution = mock ( JobExecution.class);
- when(jobExecution.getJobData()).thenReturn( jobData );
-
- exportService.doExport( exportInfo, jobExecution );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+ exportService.doExport( exportInfo, jobExecution );
}
/*Creates fake payload for testing purposes.*/
- public HashMap<String,Object> payloadBuilder() {
+ public HashMap<String, Object> payloadBuilder() {
HashMap<String, Object> payload = new HashMap<String, Object>();
Map<String, Object> properties = new HashMap<String, Object>();
Map<String, Object> storage_info = new HashMap<String, Object>();
// TODO: always put dummy values here and ignore this test.
- storage_info.put( "s3_key","insert key here" );
- storage_info.put( "s3_accessId","insert access id here");
- storage_info.put( "bucket_location","insert bucket name here");
+ storage_info.put( "s3_key", "insert key here" );
+ storage_info.put( "s3_accessId", "insert access id here" );
+ storage_info.put( "bucket_location", "insert bucket name here" );
- properties.put( "storage_provider","s3");
- properties.put( "storage_info",storage_info);
+ properties.put( "storage_provider", "s3" );
+ properties.put( "storage_info", storage_info );
- payload.put( "path", "test-organization/test-app");
- payload.put( "properties", properties);
+ payload.put( "path", "test-organization/test-app" );
+ payload.put( "properties", properties );
return payload;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/96a1a72d/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
index 432fe5e..d8a8b08 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/MockS3ExportImpl.java
@@ -10,13 +10,13 @@ import java.io.OutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
import org.apache.usergrid.management.ExportInfo;
import org.apache.usergrid.management.export.S3Export;
/**
- * Writes to file instead of s3.
- *
+ * Streams / reads the information written from the export service to a file named "test.json"
*/
public class MockS3ExportImpl implements S3Export {
@Override
@@ -28,8 +28,7 @@ public class MockS3ExportImpl implements S3Export {
//FileInputStream fis = new PrintWriter( inputStream );
try {
- outputStream = new FileOutputStream( new File("test.json") );
-
+ outputStream = new FileOutputStream( new File( "test.json" ) );
}
catch ( FileNotFoundException e ) {
e.printStackTrace();
@@ -37,10 +36,9 @@ public class MockS3ExportImpl implements S3Export {
try {
- while ( (read = (inputStream.read( bytes ))) != -1) {
+ while ( ( read = ( inputStream.read( bytes ) ) ) != -1 ) {
outputStream.write( bytes, 0, read );
}
-
}
catch ( IOException e ) {
e.printStackTrace();
[4/9] git commit: Added classes to the branch that were labeled
untracked.
Posted by sn...@apache.org.
Added classes to the branch that were labeled untracked.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e1ab9df8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e1ab9df8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e1ab9df8
Branch: refs/pull/60/merge
Commit: e1ab9df84e526364ed199dea72988f905add8393
Parents: 22a17dc
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 09:37:04 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 09:37:04 2014 -0800
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/Query.java | 5 +-
.../usergrid/persistence/entities/Export.java | 125 ++
.../usergrid/persistence/entities/Export.java | 124 --
.../rest/management/ManagementResource.java | 15 +-
.../applications/ApplicationResource.java | 84 +-
.../applications/ApplicationResource.java | 277 -----
.../rest/management/ManagementResourceIT.java | 180 ++-
.../apache/usergrid/management/ExportInfo.java | 65 +
.../usergrid/management/export/ExportJob.java | 65 +
.../management/export/ExportService.java | 37 +
.../management/export/ExportServiceImpl.java | 457 +++++++
.../usergrid/management/export/S3Export.java | 16 +
.../management/export/S3ExportImpl.java | 84 ++
.../org/usergrid/management/ExportInfo.java | 65 -
.../usergrid/management/export/ExportJob.java | 64 -
.../management/export/ExportService.java | 37 -
.../management/export/ExportServiceImpl.java | 454 -------
.../usergrid/management/export/S3Export.java | 16 -
.../management/export/S3ExportImpl.java | 84 --
.../resources/usergrid-services-context.xml | 4 +-
.../org/apache/usergrid/ServiceITSetup.java | 3 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 6 +
.../cassandra/ManagementServiceIT.java | 422 +++++++
.../management/cassandra/MockS3ExportImpl.java | 49 +
.../test/java/org/usergrid/ServiceITSetup.java | 47 -
.../java/org/usergrid/ServiceITSetupImpl.java | 135 --
.../cassandra/ManagementServiceIT.java | 1152 ------------------
.../management/cassandra/MockS3ExportImpl.java | 49 -
28 files changed, 1601 insertions(+), 2520 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 01ac231..286cd7f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -36,6 +36,9 @@ import org.antlr.runtime.TokenRewriteStream;
import org.codehaus.jackson.annotate.JsonIgnore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.persistence.Results.Level;
import org.apache.usergrid.persistence.exceptions.QueryParseException;
import org.apache.usergrid.persistence.query.tree.AndOperand;
@@ -51,8 +54,6 @@ import org.apache.usergrid.persistence.query.tree.QueryFilterLexer;
import org.apache.usergrid.persistence.query.tree.QueryFilterParser;
import org.apache.usergrid.utils.JsonUtils;
-import org.apache.commons.lang.StringUtils;
-
import static org.apache.commons.codec.binary.Base64.decodeBase64;
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang.StringUtils.split;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
new file mode 100644
index 0000000..182b2ae
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
@@ -0,0 +1,125 @@
+package org.apache.usergrid.persistence.entities;
+
+
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+
+import org.apache.usergrid.persistence.TypedEntity;
+import org.apache.usergrid.persistence.annotations.EntityProperty;
+
+
+/**
+ *Contains state information for an Entity Job
+ *
+ */
+@XmlRootElement
+public class Export extends TypedEntity {
+
+ public static enum State {
+ //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
+ PENDING,STARTED,FAILED,COMPLETED
+ }
+
+ @EntityProperty
+ protected State curState;
+
+ @EntityProperty
+ protected Long queued;
+
+ /** Time send started */
+ @EntityProperty
+ protected Long started;
+
+ /** Time processed */
+ @EntityProperty
+ protected Long finished;
+
+
+ /** Time to expire the exportJob */
+ @EntityProperty
+ protected Long expire;
+
+ /** True if exportJob is canceled */
+ @EntityProperty
+ protected Boolean canceled;
+
+ /** Error message */
+ @EntityProperty
+ protected String errorMessage;
+
+ public Export() {
+ }
+
+ public boolean isExpired () {
+ return (expire != null && expire > System.currentTimeMillis());
+ }
+
+ public Long getStarted() {
+ return started;
+ }
+
+
+ public void setStarted( final Long started ) {
+ this.started = started;
+ }
+
+
+ public Long getFinished() {
+ return finished;
+ }
+
+
+ public void setFinished( final Long finished ) {
+ this.finished = finished;
+ }
+
+
+ public Long getExpire() {
+ return expire;
+ }
+
+
+ public void setExpire( final Long expire ) {
+ this.expire = expire;
+ }
+
+
+ public Boolean getCanceled() {
+ return canceled;
+ }
+
+ //state should moved to a derived state, but it is not there yet.
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public void setState(State setter) {
+ curState = setter;
+ }
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @EntityProperty
+ public State getState() { return curState; }
+
+ public void setCanceled( final Boolean canceled ) {
+ this.canceled = canceled;
+ }
+
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+
+ public void setErrorMessage( final String errorMessage ) {
+ this.errorMessage = errorMessage;
+ }
+
+
+ public Long getQueued() {
+ return queued;
+ }
+
+
+ public void setQueued( final Long queued ) {
+ this.queued = queued;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
deleted file mode 100644
index 60d2bcb..0000000
--- a/stack/core/src/main/java/org/usergrid/persistence/entities/Export.java
+++ /dev/null
@@ -1,124 +0,0 @@
-package org.usergrid.persistence.entities;
-
-
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.codehaus.jackson.map.annotate.JsonSerialize;
-import org.usergrid.persistence.TypedEntity;
-import org.usergrid.persistence.annotations.EntityProperty;
-
-
-/**
- *Contains state information for an Entity Job
- *
- */
-@XmlRootElement
-public class Export extends TypedEntity {
-
- public static enum State {
- //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
- PENDING,STARTED,FAILED,COMPLETED
- }
-
- @EntityProperty
- protected State curState;
-
- @EntityProperty
- protected Long queued;
-
- /** Time send started */
- @EntityProperty
- protected Long started;
-
- /** Time processed */
- @EntityProperty
- protected Long finished;
-
-
- /** Time to expire the exportJob */
- @EntityProperty
- protected Long expire;
-
- /** True if exportJob is canceled */
- @EntityProperty
- protected Boolean canceled;
-
- /** Error message */
- @EntityProperty
- protected String errorMessage;
-
- public Export() {
- }
-
- public boolean isExpired () {
- return (expire != null && expire > System.currentTimeMillis());
- }
-
- public Long getStarted() {
- return started;
- }
-
-
- public void setStarted( final Long started ) {
- this.started = started;
- }
-
-
- public Long getFinished() {
- return finished;
- }
-
-
- public void setFinished( final Long finished ) {
- this.finished = finished;
- }
-
-
- public Long getExpire() {
- return expire;
- }
-
-
- public void setExpire( final Long expire ) {
- this.expire = expire;
- }
-
-
- public Boolean getCanceled() {
- return canceled;
- }
-
- //state should moved to a derived state, but it is not there yet.
- @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- @EntityProperty
- public void setState(State setter) {
- curState = setter;
- }
- @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
- @EntityProperty
- public State getState() { return curState; }
-
- public void setCanceled( final Boolean canceled ) {
- this.canceled = canceled;
- }
-
-
- public String getErrorMessage() {
- return errorMessage;
- }
-
-
- public void setErrorMessage( final String errorMessage ) {
- this.errorMessage = errorMessage;
- }
-
-
- public Long getQueued() {
- return queued;
- }
-
-
- public void setQueued( final Long queued ) {
- this.queued = queued;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
index 59ceb4e..72e967f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/ManagementResource.java
@@ -37,6 +37,14 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.error.OAuthError;
+import org.apache.amber.oauth2.common.exception.OAuthProblemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.amber.oauth2.common.message.types.GrantType;
+import org.apache.commons.lang.StringUtils;
+import org.apache.shiro.codec.Base64;
+
import org.apache.usergrid.management.UserInfo;
import org.apache.usergrid.management.exceptions.DisabledAdminUserException;
import org.apache.usergrid.management.exceptions.UnactivatedAdminUserException;
@@ -48,13 +56,6 @@ import org.apache.usergrid.rest.management.users.UsersResource;
import org.apache.usergrid.security.oauth.AccessInfo;
import org.apache.usergrid.security.shiro.utils.SubjectUtils;
-import org.apache.amber.oauth2.common.error.OAuthError;
-import org.apache.amber.oauth2.common.exception.OAuthProblemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.amber.oauth2.common.message.types.GrantType;
-import org.apache.commons.lang.StringUtils;
-import org.apache.shiro.codec.Base64;
-
import com.sun.jersey.api.view.Viewable;
import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
index dff0924..5f45326 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/ApplicationResource.java
@@ -16,6 +16,7 @@
package org.apache.usergrid.rest.management.organizations.applications;
+import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
@@ -25,30 +26,44 @@ import javax.ws.rs.DefaultValue;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import javax.ws.rs.core.UriInfo;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;
+
+import org.apache.amber.oauth2.common.exception.OAuthSystemException;
+import org.apache.amber.oauth2.common.message.OAuthResponse;
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.management.ApplicationInfo;
+import org.apache.usergrid.management.ExportInfo;
import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.rest.AbstractContextResource;
import org.apache.usergrid.rest.ApiResponse;
+import org.apache.usergrid.rest.applications.ServiceResource;
import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
+import org.apache.usergrid.rest.utils.JSONPUtils;
import org.apache.usergrid.security.oauth.ClientCredentialsInfo;
import org.apache.usergrid.security.providers.SignInAsProvider;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.services.ServiceManager;
-import org.apache.commons.lang.StringUtils;
-
import com.google.common.base.Preconditions;
import com.sun.jersey.api.json.JSONWithPadding;
+import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
+import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
+import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
+import static javax.servlet.http.HttpServletResponse.SC_OK;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
@@ -60,6 +75,8 @@ import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
})
public class ApplicationResource extends AbstractContextResource {
+ @Autowired
+ protected ExportService exportService;
OrganizationInfo organization;
UUID applicationId;
ApplicationInfo application;
@@ -192,4 +209,67 @@ public class ApplicationResource extends AbstractContextResource {
return new JSONWithPadding( response, callback );
}
+
+ @POST
+ @Path("export")
+ @Consumes(APPLICATION_JSON)
+ @RequireOrganizationAccess
+ public Response exportPostJson (@Context UriInfo ui,
+ Map<String, Object> json,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
+ throws OAuthSystemException {
+
+
+ OAuthResponse response = null;
+ UUID jobUUID = null;
+ Map<String, String> uuidRet = new HashMap<String, String>( );
+
+ try {
+ //parse the json into some useful object (the config params)
+ ExportInfo objEx = new ExportInfo(json);
+ objEx.setApplicationId( applicationId );
+ jobUUID = exportService.schedule(objEx);
+ uuidRet.put( "jobUUID", jobUUID.toString() );
+
+ }
+ catch (NullPointerException e) {
+ OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
+ .setErrorDescription( "Job Not Created" )
+ .buildJSONMessage();
+
+ return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
+ .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
+ }
+ catch (Exception e) {
+ //TODO:throw descriptive error message and or include on in the response
+ //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
+ return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
+ }
+
+ return Response.status(SC_ACCEPTED).entity(uuidRet).build();
+
+ //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
+ // .entity( wrapWithCallback( "", callback ) ).build();
+ }
+
+ @GET
+ @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
+ public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
+ @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
+
+ Export entity;
+ try {
+ entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
+ }catch(Exception e) {
+ return Response.status( SC_BAD_REQUEST ).build();
+ }
+ //validate this user owns it
+
+ if (entity == null) {
+ return Response.status(SC_BAD_REQUEST).build();
+ }
+
+ return Response.status(SC_OK).entity(entity.getState()).build();
+ //return Response.status(SC_OK).entity(state).build();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java b/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
deleted file mode 100644
index 5a29500..0000000
--- a/stack/rest/src/main/java/org/usergrid/rest/management/organizations/applications/ApplicationResource.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*******************************************************************************
- * Copyright 2012 Apigee Corporation
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.usergrid.rest.management.organizations.applications;
-
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.annotation.Scope;
-import org.springframework.stereotype.Component;
-import org.usergrid.management.ApplicationInfo;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.management.export.ExportService;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.rest.AbstractContextResource;
-import org.usergrid.rest.ApiResponse;
-import org.usergrid.rest.applications.ServiceResource;
-import org.usergrid.rest.security.annotations.RequireOrganizationAccess;
-import org.usergrid.rest.utils.JSONPUtils;
-import org.usergrid.security.oauth.ClientCredentialsInfo;
-import org.usergrid.security.providers.SignInAsProvider;
-import org.usergrid.security.providers.SignInProviderFactory;
-import org.usergrid.services.ServiceManager;
-
-import org.apache.amber.oauth2.common.exception.OAuthSystemException;
-import org.apache.amber.oauth2.common.message.OAuthResponse;
-import org.apache.commons.lang.StringUtils;
-
-import org.apache.usergrid.rest.security.annotations.RequireOrganizationAccess;
-
-import com.google.common.base.Preconditions;
-import com.sun.jersey.api.json.JSONWithPadding;
-
-import static javax.servlet.http.HttpServletResponse.SC_ACCEPTED;
-import static javax.servlet.http.HttpServletResponse.SC_BAD_REQUEST;
-import static javax.servlet.http.HttpServletResponse.SC_INTERNAL_SERVER_ERROR;
-import static javax.servlet.http.HttpServletResponse.SC_OK;
-import static javax.ws.rs.core.MediaType.APPLICATION_JSON;
-
-
-@Component("org.usergrid.rest.management.organizations.applications.ApplicationResource")
-@Scope("prototype")
-@Produces({
- MediaType.APPLICATION_JSON, "application/javascript", "application/x-javascript", "text/ecmascript",
- "application/ecmascript", "text/jscript"
-})
-public class ApplicationResource extends AbstractContextResource {
-
- @Autowired
- protected ExportService exportService;
- OrganizationInfo organization;
- UUID applicationId;
- ApplicationInfo application;
-
- @Autowired
- private SignInProviderFactory signInProviderFactory;
-
-
- public ApplicationResource() {
- }
-
-
- public ApplicationResource init( OrganizationInfo organization, UUID applicationId ) {
- this.organization = organization;
- this.applicationId = applicationId;
- return this;
- }
-
-
- public ApplicationResource init( OrganizationInfo organization, ApplicationInfo application ) {
- this.organization = organization;
- applicationId = application.getId();
- this.application = application;
- return this;
- }
-
-
- @RequireOrganizationAccess
- @DELETE
- public JSONWithPadding deleteApplicationFromOrganizationByApplicationId( @Context UriInfo ui,
- @QueryParam("callback")
- @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "delete application from organization" );
-
- management.deleteOrganizationApplication( organization.getUuid(), applicationId );
-
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @GET
- public JSONWithPadding getApplication( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- ServiceManager sm = smf.getServiceManager( applicationId );
- response.setAction( "get" );
- response.setApplication( sm.getApplication() );
- response.setParams( ui.getQueryParameters() );
- response.setResults( management.getApplicationMetadata( applicationId ) );
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @GET
- @Path("credentials")
- public JSONWithPadding getCredentials( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "get application client credentials" );
-
- ClientCredentialsInfo credentials =
- new ClientCredentialsInfo( management.getClientIdForApplication( applicationId ),
- management.getClientSecretForApplication( applicationId ) );
-
- response.setCredentials( credentials );
- return new JSONWithPadding( response, callback );
- }
-
-
- @RequireOrganizationAccess
- @POST
- @Path("credentials")
- public JSONWithPadding generateCredentials( @Context UriInfo ui,
- @QueryParam("callback") @DefaultValue("callback") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "generate application client credentials" );
-
- ClientCredentialsInfo credentials =
- new ClientCredentialsInfo( management.getClientIdForApplication( applicationId ),
- management.newClientSecretForApplication( applicationId ) );
-
- response.setCredentials( credentials );
- return new JSONWithPadding( response, callback );
- }
-
-
- @POST
- @Path("sia-provider")
- @Consumes(APPLICATION_JSON)
- @RequireOrganizationAccess
- public JSONWithPadding configureProvider( @Context UriInfo ui, @QueryParam("provider_key") String siaProvider,
- Map<String, Object> json,
- @QueryParam("callback") @DefaultValue("") String callback )
- throws Exception {
-
- ApiResponse response = createApiResponse();
- response.setAction( "post signin provider configuration" );
-
- Preconditions.checkArgument( siaProvider != null, "Sign in provider required" );
-
- SignInAsProvider signInAsProvider = null;
- if ( StringUtils.equalsIgnoreCase( siaProvider, "facebook" ) ) {
- signInAsProvider =
- signInProviderFactory.facebook( smf.getServiceManager( applicationId ).getApplication() );
- }
- else if ( StringUtils.equalsIgnoreCase( siaProvider, "pingident" ) ) {
- signInAsProvider =
- signInProviderFactory.pingident( smf.getServiceManager( applicationId ).getApplication() );
- }
- else if ( StringUtils.equalsIgnoreCase( siaProvider, "foursquare" ) ) {
- signInAsProvider =
- signInProviderFactory.foursquare( smf.getServiceManager( applicationId ).getApplication() );
- }
-
- Preconditions
- .checkArgument( signInAsProvider != null, "No signin provider found by that name: " + siaProvider );
-
- signInAsProvider.saveToConfiguration( json );
-
- return new JSONWithPadding( response, callback );
- }
- //add export here
- @POST
- @Path("export")
- @Consumes(APPLICATION_JSON)
- @RequireOrganizationAccess
- public Response exportPostJson (@Context UriInfo ui,
- Map<String, Object> json,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback)
- throws OAuthSystemException {
-
-
- OAuthResponse response = null;
- UUID jobUUID = null;
- Map<String, String> uuidRet = new HashMap<String, String>( );
-
- try {
- //parse the json into some useful object (the config params)
- ExportInfo objEx = new ExportInfo(json);
- objEx.setApplicationId( applicationId );
- jobUUID = exportService.schedule(objEx);
- uuidRet.put( "jobUUID", jobUUID.toString() );
-
- }
- catch (NullPointerException e) {
- OAuthResponse errorMsg = OAuthResponse.errorResponse( SC_BAD_REQUEST )
- .setErrorDescription( "Job Not Created" )
- .buildJSONMessage();
-
- return Response.status( errorMsg.getResponseStatus() ).type( JSONPUtils.jsonMediaType( callback ) )
- .entity( ServiceResource.wrapWithCallback( errorMsg.getBody(), callback ) ).build();
- }
- catch (Exception e) {
- //TODO:throw descriptive error message and or include on in the response
- //TODO:fix below, it doesn't work if there is an exception. Make it look like the OauthResponse.
- return Response.status( SC_INTERNAL_SERVER_ERROR ).build();
- }
-
- return Response.status(SC_ACCEPTED).entity(uuidRet).build();
-
- //Response.status( response.getResponseStatus() ).type( jsonMediaType( callback ) )
- // .entity( wrapWithCallback( "", callback ) ).build();
- }
-
- @GET
- @Path( "export/{jobUUID: [A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}}" )
- public Response exportGetJson(@Context UriInfo ui,@PathParam( "jobUUID" ) UUID jobUUIDStr,
- @QueryParam( "callback" ) @DefaultValue( "" ) String callback ) throws Exception {
-
- Export entity;
- try {
- entity = smf.getServiceManager( applicationId ).getEntityManager().get(jobUUIDStr, Export.class );
- }catch(Exception e) {
- return Response.status( SC_BAD_REQUEST ).build();
- }
- //validate this user owns it
-
- if (entity == null) {
- return Response.status(SC_BAD_REQUEST).build();
- }
-
- return Response.status(SC_OK).entity(entity.getState()).build();
- //return Response.status(SC_OK).entity(state).build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index b379770..0621219 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -18,28 +18,31 @@ package org.apache.usergrid.rest.management;
import java.util.HashMap;
import java.util.Map;
+import java.util.UUID;
import javax.ws.rs.core.MediaType;
import org.codehaus.jackson.JsonNode;
+import org.junit.Ignore;
import org.junit.Test;
+
+import org.apache.commons.lang.StringUtils;
+
import org.apache.usergrid.cassandra.Concurrent;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.OrganizationOwnerInfo;
import org.apache.usergrid.rest.AbstractRestIT;
import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
-import org.apache.commons.lang.StringUtils;
-
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.representation.Form;
+import static org.apache.usergrid.utils.MapUtils.hashMap;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.apache.usergrid.utils.MapUtils.hashMap;
/** @author tnine */
@@ -528,4 +531,175 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, status );
}
+ @Test
+ public void exportCallSuccessful() throws Exception {
+ Status responseStatus = Status.OK;
+ JsonNode node = null;
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.OK, responseStatus );
+ }
+
+ @Ignore
+ public void exportCallCreationEntities100() throws Exception {
+ Status responseStatus = Status.OK;
+ JsonNode node = null;
+
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ //TODO: make sure to put a valid admin token here.
+ storage_info.put( "admin_token", adminToken() );
+ //TODO: always put dummy values here and ignore this test.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app/user");
+ payload.put( "properties", properties);
+
+ for (int i = 0; i < 100; i++) {
+ Map<String, String> userCreation = hashMap( "type", "app_user" ).map( "name", "fred"+i );
+
+ node = resource().path( "/test-organization/test-app/app_users" )
+ .queryParam( "access_token", access_token ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, userCreation );
+ }
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.OK, responseStatus );
+ }
+
+ @Test
+ public void exportUUIDRetTest() throws Exception {
+ Status responseStatus = Status.ACCEPTED;
+ String uuid;
+ UUID jobUUID = null;
+ JsonNode node = null;
+
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+
+ }
+ catch ( UniformInterfaceException uie ) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+ assertEquals( Status.ACCEPTED, responseStatus );
+ assertNotNull( node.get( "jobUUID" ) );
+ }
+ /*Make a test with an invalid uuid and a wrong uuid.*/
+ //all tests should be moved to OrganizationResourceIT ( *not* Organizations there is a difference)
+ @Test
+ public void exportGetJobStatTest() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+
+ HashMap<String, Object> payload = payloadBuilder();
+
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export" )
+ .queryParam( "access_token", superAdminToken() )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).post( JsonNode.class, payload );
+ String uuid = String.valueOf( node.get( "jobUUID" ) );
+ uuid = uuid.replaceAll( "\"","" );
+
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/"+uuid).queryParam( "access_token",
+ superAdminToken() ).accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE ).get( JsonNode.class );
+ // node = resource().path( "/management/exportStats/"+uuid ).accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+ // .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+
+
+ assertEquals( Status.OK, responseStatus );
+ assertEquals( "PENDING",node.asText() );//TODO: do tests for other states in service tier
+
+ }
+ //do an unauthorized test for both post and get
+ @Test
+ public void exportGetWrongUUID() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+ try {
+ node = resource().path(
+ "/management/orgs/test-organization/apps/test-app/export/"+fake ).queryParam( "access_token",
+ superAdminToken() ).accept( MediaType.APPLICATION_JSON ).type(
+ MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+ assertEquals(Status.BAD_REQUEST, responseStatus);
+
+ }
+
+ @Test
+ public void exportGetUnauthorized() throws Exception {
+ JsonNode node = null;
+ Status responseStatus = Status.OK;
+ UUID fake = UUID.fromString( "AAAAAAAA-FFFF-FFFF-FFFF-AAAAAAAAAAAA" );
+ try {
+ node = resource().path( "/management/orgs/test-organization/apps/test-app/export/" + fake )
+ .accept( MediaType.APPLICATION_JSON )
+ .type( MediaType.APPLICATION_JSON_TYPE )
+ .get( JsonNode.class );
+ }catch(UniformInterfaceException uie) {
+ responseStatus = uie.getResponse().getClientResponseStatus();
+ }
+ assertEquals(Status.UNAUTHORIZED, responseStatus);
+ }
+
+ /*Creates fake payload for testing purposes.*/
+ public HashMap<String,Object> payloadBuilder() {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ //TODO: make sure to put a valid admin token here.
+ //storage_info.put( "admin_token","insert_token_data_here" );
+ //TODO: always put dummy values here and ignore this test.
+ //TODO: add a ret for when s3 values are invalid.
+ storage_info.put( "s3_key","insert key here" );
+ storage_info.put( "s3_accessId","insert access id here");
+ storage_info.put( "bucket_location","insert bucket name here");
+
+
+ properties.put( "storage_provider","s3");
+ properties.put( "storage_info",storage_info);
+
+ payload.put( "path", "test-organization/test-app");
+ payload.put( "properties", properties);
+ return payload;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
new file mode 100644
index 0000000..f4ed4cf
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/ExportInfo.java
@@ -0,0 +1,65 @@
+package org.apache.usergrid.management;
+
+
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.TypedEntity;
+
+
+/**
+ * Created by ApigeeCorporation on 1/31/14.
+ */
+//TODO: Documentation on this class.
+public class ExportInfo extends TypedEntity {
+
+ private String path;
+ private Map<String, Object> properties;
+ private String storage_provider;
+ private Map<String, Object> storage_info;
+ private String s3_accessId;
+ private String s3_key;
+ private String bucket_location;
+ private UUID applicationId;
+
+ public ExportInfo ( Map<String, Object> exportData) {
+ path = (String) exportData.get("path");
+ properties = (Map) exportData.get("properties");
+ storage_provider = (String) properties.get ("storage_provider");
+ storage_info = (Map) properties.get("storage_info");
+ s3_accessId = (String) storage_info.get("s3_accessId");
+ s3_key = (String) storage_info.get("s3_key");
+ bucket_location = (String) storage_info.get("bucket_location");
+ }
+
+
+ public UUID getApplicationId() {
+ return applicationId;
+ }
+
+
+ public String getPath () {
+ return path;
+ }
+
+ //Wouldn't get exposed.
+ public Map<String, Object> getProperties() {
+ return properties;
+ }
+
+ public String getStorage_provider () {
+ return storage_provider;
+ }
+ //TODO: write setter methods
+
+ public Map<String, Object> getStorage_info () { return storage_info; }
+
+ //TODO: is this a security concern? How would we get rid of the key once we're done with this value?
+ public String getS3_key () { return s3_key; }
+
+ public String getBucket_location () { return bucket_location; }
+
+ public String getS3_accessId () { return s3_accessId; }
+
+ public void setApplicationId (UUID appId) { applicationId = appId;}
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
new file mode 100644
index 0000000..9830040
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
@@ -0,0 +1,65 @@
+package org.apache.usergrid.management.export;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.job.OnlyOnceJob;
+import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.persistence.entities.JobData;
+
+
+/**
+ *
+ *you could make an enum here, that contains the state info
+ * look at scotts code and emulate that to see wha tyou can return in the json object
+ */
+@Component("exportJob")
+public class ExportJob extends OnlyOnceJob {
+ private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
+
+ @Autowired
+ ExportService exportService;
+
+ public ExportJob() {
+ logger.info("ExportJob created " + this);
+ }
+
+ @Override
+ public void doJob(JobExecution jobExecution) throws Exception {
+ logger.info( "execute ExportJob {}", jobExecution );
+
+ ExportInfo config = null;
+ //add check for null jobData.
+ JobData jobData = jobExecution.getJobData();
+ config = (ExportInfo) jobData.getProperty( "exportInfo" );
+ if(jobData == null) {
+ logger.error( "jobData cannot be null" );
+ return;
+ }
+ else if (config == null) {
+ logger.error( "Export information cannot be null" );
+ return;
+ }
+ else {
+ jobExecution.heartbeat();
+ exportService.doExport( config,jobExecution );
+ }
+ logger.info( "executed ExportJob process completed" );
+ }
+
+ @Override
+ protected long getDelay(JobExecution jobExecution) throws Exception {
+ //return arbitrary number
+ return 100;
+ }
+
+
+ @Autowired
+ public void setExportService( final ExportService exportService ) {
+ this.exportService = exportService;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
new file mode 100644
index 0000000..6e9f4e5
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportService.java
@@ -0,0 +1,37 @@
+package org.apache.usergrid.management.export;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.management.ExportInfo;
+
+
+/**
+ * Performs all functions related to exporting
+ *
+ */
+public interface ExportService {
+
+ /**
+ * Schedules the export to execute
+ * @param config
+ */
+ UUID schedule(ExportInfo config) throws Exception;
+
+
+ /**
+ * Perform the export to the external resource
+ * @param config
+ */
+ void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
+
+ /**
+ * Returns the current state of the service.
+ * @return
+ */
+ String getState(UUID appId,UUID state) throws Exception;
+
+ void setS3Export(S3Export s3Export);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
new file mode 100644
index 0000000..2d8ff5c
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -0,0 +1,457 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+
+
+import org.apache.usergrid.batch.JobExecution;
+import org.apache.usergrid.batch.service.SchedulerService;
+import org.apache.usergrid.management.ExportInfo;
+
+import org.apache.usergrid.management.ManagementService;
+import org.apache.usergrid.management.OrganizationInfo;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.PagingResultsIterator;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.entities.Export;
+import org.apache.usergrid.persistence.entities.JobData;
+
+import com.google.common.collect.BiMap;
+
+
+/**
+ *
+ *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
+ * it will just be the one app and the one org and all of it's collections.
+ */
+public class ExportServiceImpl implements ExportService {
+
+
+ private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+ public static final String EXPORT_ID = "exportId";
+ public static final String EXPORT_JOB_NAME = "exportJob";
+ //dependency injection
+ private SchedulerService sch;
+
+ //injected the Entity Manager Factory
+ protected EntityManagerFactory emf;
+
+ //inject Management Service to access Organization Data
+ private ManagementService managementService;
+
+ //Maximum amount of entities retrieved in a single go.
+ public static final int MAX_ENTITY_FETCH = 100;
+
+ //Amount of time that has passed before sending another heart beat in millis
+ public static final int TIMESTAMP_DELTA = 5000;
+
+ private JsonFactory jsonFactory = new JsonFactory();
+
+ protected long startTime = System.currentTimeMillis();
+
+ private S3Export s3Export;
+
+ @Override
+ public UUID schedule( final ExportInfo config ) throws Exception {
+
+ EntityManager em = emf.getEntityManager( config.getApplicationId());
+
+ Export export = new Export();
+ export.setState( Export.State.PENDING );
+
+ //validate that org exists,then app, then collection.
+
+ String pathToBeParsed = config.getPath();
+ //split the path so that you can verify that the organization and the app exist.
+ String[] pathItems = pathToBeParsed.split( "/" );
+
+
+ try {
+ managementService.getOrganizationByName( pathItems[0] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Organization doesn't exist" );
+ }
+
+ try {
+ managementService.getApplicationInfo( pathItems[1] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Application doesn't exist" );
+ }
+
+
+ //TODO: parse path and make sure all the things you need actually exist. then throw
+ // good error messages when not found.
+
+ //write to the em
+ export = em.create( export );
+ export.setState( Export.State.PENDING );
+ em.update( export );
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "exportInfo", config );
+ jobData.setProperty( EXPORT_ID, export.getUuid() );
+
+ long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
+
+ sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
+
+ return export.getUuid();
+
+ }
+ /**
+ * Query Entity Manager for specific Export Entity within application
+ * @param appId,uuid
+ * @return String
+ * @throws Exception
+ */
+ @Override
+ public String getState(final UUID appId, final UUID uuid) throws Exception {
+
+ EntityManager rootEm = emf.getEntityManager( appId );
+ Export export = rootEm.get( uuid, Export.class );
+
+ if(export == null){
+ return null;
+ }
+ return export.getState().toString();
+ }
+
+
+ @Override
+ public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
+
+ UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+ EntityManager em = emf.getEntityManager( config.getApplicationId());
+ Export export = em.get( exportId, Export.class );
+
+ String pathToBeParsed = config.getPath();
+ String[] pathItems = pathToBeParsed.split( "/" );
+ try {
+ managementService.getOrganizationByName( pathItems[0] );
+ }
+ catch ( Exception e ) {
+ logger.error( "Organization doesn't exist" );
+ return;
+ }
+
+ //update state and re-write the entity
+ export.setState( Export.State.STARTED );
+
+ em.update( export );
+
+ Map<UUID, String> organizationGet = getOrgs(config);
+ for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
+ //needs to pass app name, and possibly collection to export
+ exportApplicationsForOrg( organization, config, jobExecution );
+ }
+ export.setState( Export.State.COMPLETED );
+ em.update( export );
+ }
+
+ private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
+ // Loop through the organizations
+ // TODO:this will come from the orgs in schedule when you do the validations. delete orgId
+ UUID orgId = null;
+
+ Map<UUID, String> organizationNames = null;
+
+ if ( orgId == null ) {
+ organizationNames = managementService.getOrganizations();
+ }
+
+ else {
+ OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
+
+ if ( info == null ) {
+ logger.error( "Organization info is null!" );
+ }
+
+ organizationNames = new HashMap<UUID, String>();
+ organizationNames.put( orgId, info.getName() );
+ }
+
+
+ return organizationNames;
+ }
+
+
+ public SchedulerService getSch() {
+ return sch;
+ }
+
+
+ public void setSch( final SchedulerService sch ) {
+ this.sch = sch;
+ }
+
+
+ public EntityManagerFactory getEmf() {
+ return emf;
+ }
+
+
+ public void setEmf( final EntityManagerFactory emf ) {
+ this.emf = emf;
+ }
+
+
+ public ManagementService getManagementService() {
+
+ return managementService;
+ }
+
+ public void setManagementService( final ManagementService managementService ) {
+ this.managementService = managementService;
+ }
+
+ //write test checking to see what happens if the input stream is closed or wrong.
+ //TODO: make multipart streaming functional
+ //currently only stores the collection in memory then flushes it.
+ private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
+ final JobExecution jobExecution ) throws Exception {
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+
+ logger.info( "" + organization );
+
+ // Loop through the applications per organization
+ BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organization.getKey() );
+ for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
+
+ logger.info( application.getValue() + " : " + application.getKey() );
+
+ String appFileName = prepareOutputFileName( "application", application.getValue() );
+
+ JsonGenerator jg = getJsonGenerator( baos );
+
+ EntityManager em = emf.getEntityManager( application.getKey() );
+
+ jg.writeStartArray();
+
+ Map<String, Object> metadata = em.getApplicationCollectionMetadata();
+ long starting_time = System.currentTimeMillis();
+
+ // Loop through the collections. This is the only way to loop
+ // through the entities in the application (former namespace).
+ //could support queries, just need to implement that in the rest endpoint.
+ for ( String collectionName : metadata.keySet() ) {
+ if(collectionName.equals( "exports" )) {
+ continue;
+ }
+
+ Query query = new Query();
+ query.setLimit( MAX_ENTITY_FETCH );
+ query.setResultsLevel( Results.Level.ALL_PROPERTIES );
+ Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
+
+ PagingResultsIterator itr = new PagingResultsIterator( entities );
+
+ for( Object e: itr){
+ starting_time = checkTimeDelta( starting_time, jobExecution );
+ Entity entity = ( Entity ) e;
+
+ jg.writeStartObject();
+ jg.writeFieldName( "Metadata" );
+ jg.writeObject( entity );
+ saveCollectionMembers( jg, em, application.getValue(), entity );
+ jg.writeEndObject();
+ }
+ }
+
+ // Close writer and file for this application.
+ jg.writeEndArray();
+ jg.close();
+ baos.flush();
+ baos.close();
+
+ InputStream is = new ByteArrayInputStream( baos.toByteArray() );
+ s3Export.copyToS3( is, config , appFileName);
+ }
+ }
+
+
+ public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) {
+
+ long cur_time = System.currentTimeMillis();
+
+ if ( startingTime <= ( cur_time - TIMESTAMP_DELTA ) ) {
+ jobExecution.heartbeat();
+ return cur_time;
+ }
+ return startingTime;
+ }
+
+
+ /**
+ * Serialize and save the collection members of this <code>entity</code>
+ *
+ * @param em Entity Manager
+ * @param application Application name
+ * @param entity entity
+ */
+ private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity )
+ throws Exception {
+
+ Set<String> collections = em.getCollections( entity );
+ //jg.writeStartObject();
+
+ // Only create entry for Entities that have collections
+ if ( ( collections == null ) || collections.isEmpty() ) {
+ return;
+ }
+
+
+ for ( String collectionName : collections ) {
+
+ jg.writeFieldName( collectionName );
+ jg.writeStartArray();
+
+ //is 100000 an arbitary number?
+ Results collectionMembers =
+ em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
+
+ List<UUID> entityIds = collectionMembers.getIds();
+
+ if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
+ for ( UUID childEntityUUID : entityIds ) {
+ jg.writeObject( childEntityUUID.toString() );
+ }
+ }
+
+ // End collection array.
+ jg.writeEndArray();
+ }
+
+ // Write connections
+ saveConnections( entity, em, jg );
+
+ // Write dictionaries
+ saveDictionaries( entity, em, jg );
+
+ // End the object if it was Started
+ //jg.writeEndObject();
+ }
+
+
+ /**
+ * Persists the connection for this entity.
+ */
+ private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "dictionaries" );
+ jg.writeStartObject();
+
+ Set<String> dictionaries = em.getDictionaries( entity );
+ for ( String dictionary : dictionaries ) {
+
+ Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
+
+ // nothing to do
+ if ( dict.isEmpty() ) {
+ continue;
+ }
+
+ jg.writeFieldName( dictionary );
+
+ jg.writeStartObject();
+
+ for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
+ jg.writeFieldName( entry.getKey().toString() );
+ jg.writeObject( entry.getValue() );
+ }
+
+ jg.writeEndObject();
+ }
+ jg.writeEndObject();
+ }
+
+
+ /**
+ * Persists the connection for this entity.
+ */
+ private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
+
+ jg.writeFieldName( "connections" );
+ jg.writeStartObject();
+
+ Set<String> connectionTypes = em.getConnectionTypes( entity );
+ for ( String connectionType : connectionTypes ) {
+
+ jg.writeFieldName( connectionType );
+ jg.writeStartArray();
+
+ Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Results.Level.IDS );
+ List<ConnectionRef> connections = results.getConnections();
+
+ for ( ConnectionRef connectionRef : connections ) {
+ jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
+ }
+
+ jg.writeEndArray();
+ }
+ jg.writeEndObject();
+ }
+
+
+ protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
+ //TODO:shouldn't the below be UTF-16?
+ //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
+
+ JsonGenerator jg = jsonFactory.createJsonGenerator( out );
+ jg.setPrettyPrinter( new DefaultPrettyPrinter() );
+ jg.setCodec( new ObjectMapper() );
+ return jg;
+ }
+
+
+ protected File createOutputFile( String type, String name ) {
+ return new File( prepareOutputFileName( type, name ) );
+ }
+
+
+ /**
+ * @param type just a label such us: organization, application.
+ *
+ * @return the file name concatenated with the type and the name of the collection
+ */
+ protected String prepareOutputFileName( String type, String name ) {
+ StringBuilder str = new StringBuilder();
+ str.append( name );
+ str.append( "." );
+ str.append( startTime );
+ str.append( ".json" );
+
+ String outputFileName = str.toString();
+ //TODO:this is , i feel, bad practice so make sure to come back here and fix it up.
+
+ return outputFileName;
+ }
+
+
+ @Autowired
+ @Override
+ public void setS3Export( S3Export s3Export ) { this.s3Export = s3Export; }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
new file mode 100644
index 0000000..941a612
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3Export.java
@@ -0,0 +1,16 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.InputStream;
+
+import org.apache.usergrid.management.ExportInfo;
+
+
+/**
+ *
+ *
+ */
+public interface S3Export {
+ void copyToS3( InputStream inputStream, ExportInfo exportInfo, String filename );
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
new file mode 100644
index 0000000..54d8ab1
--- /dev/null
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/S3ExportImpl.java
@@ -0,0 +1,84 @@
+package org.apache.usergrid.management.export;
+
+
+import java.io.InputStream;
+import java.util.Properties;
+
+import org.jclouds.ContextBuilder;
+import org.jclouds.blobstore.AsyncBlobStore;
+import org.jclouds.blobstore.BlobStoreContext;
+import org.jclouds.blobstore.domain.Blob;
+import org.jclouds.blobstore.domain.BlobBuilder;
+import org.jclouds.blobstore.options.PutOptions;
+import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+import org.jclouds.netty.config.NettyPayloadModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.apache.usergrid.management.ExportInfo;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.inject.Module;
+
+
+/**
+ *
+ *
+ */
+public class S3ExportImpl implements S3Export {
+
+ @Override
+ public void copyToS3( final InputStream inputStream, final ExportInfo exportInfo, String filename ) {
+
+ Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
+ /*won't need any of the properties as I have the export info*/
+ String bucketName = exportInfo.getBucket_location();
+ String accessId = exportInfo.getS3_accessId();
+ String secretKey = exportInfo.getS3_key();
+
+ Properties overrides = new Properties();
+ overrides.setProperty( "s3" + ".identity", accessId );
+ overrides.setProperty( "s3" + ".credential", secretKey );
+
+ final Iterable<? extends Module> MODULES = ImmutableSet
+ .of( new JavaUrlHttpCommandExecutorServiceModule(), new Log4JLoggingModule(), new NettyPayloadModule
+ () );
+
+ BlobStoreContext context =
+ ContextBuilder.newBuilder( "s3" ).credentials( accessId, secretKey ).modules( MODULES )
+ .overrides( overrides ).buildView( BlobStoreContext.class );
+
+ // Create Container (the bucket in s3)
+ try {
+ AsyncBlobStore blobStore = context.getAsyncBlobStore(); // it can be changed to sync
+ // BlobStore (returns false if it already exists)
+ ListenableFuture<Boolean> container = blobStore.createContainerInLocation( null, bucketName );
+ if ( container.get() ) {
+ logger.info( "Created bucket " + bucketName );
+ }
+ }
+ catch ( Exception ex ) {
+ logger.error( "Could not start binary service: {}", ex.getMessage() );
+ return;
+ }
+
+ try {
+ AsyncBlobStore blobStore = context.getAsyncBlobStore();
+ BlobBuilder blobBuilder =
+ blobStore.blobBuilder( filename ).payload( inputStream ).calculateMD5().contentType( "text/plain" );
+
+
+ Blob blob = blobBuilder.build();
+
+ ListenableFuture<String> futureETag = blobStore.putBlob( bucketName, blob, PutOptions.Builder.multipart() );
+
+ logger.info( "Uploaded file etag=" + futureETag.get() );
+ }
+ catch ( Exception e ) {
+ logger.error( "Error uploading to blob store", e );
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java b/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
deleted file mode 100644
index 7821ca6..0000000
--- a/stack/services/src/main/java/org/usergrid/management/ExportInfo.java
+++ /dev/null
@@ -1,65 +0,0 @@
-package org.usergrid.management;
-
-
-import java.util.Map;
-import java.util.UUID;
-
-import org.usergrid.persistence.TypedEntity;
-
-
-/**
- * Created by ApigeeCorporation on 1/31/14.
- */
-//TODO: Documentation on this class.
-public class ExportInfo extends TypedEntity {
-
- private String path;
- private Map<String, Object> properties;
- private String storage_provider;
- private Map<String, Object> storage_info;
- private String s3_accessId;
- private String s3_key;
- private String bucket_location;
- private UUID applicationId;
-
- public ExportInfo ( Map<String, Object> exportData) {
- path = (String) exportData.get("path");
- properties = (Map) exportData.get("properties");
- storage_provider = (String) properties.get ("storage_provider");
- storage_info = (Map) properties.get("storage_info");
- s3_accessId = (String) storage_info.get("s3_accessId");
- s3_key = (String) storage_info.get("s3_key");
- bucket_location = (String) storage_info.get("bucket_location");
- }
-
-
- public UUID getApplicationId() {
- return applicationId;
- }
-
-
- public String getPath () {
- return path;
- }
-
- //Wouldn't get exposed.
- public Map<String, Object> getProperties() {
- return properties;
- }
-
- public String getStorage_provider () {
- return storage_provider;
- }
- //TODO: write setter methods
-
- public Map<String, Object> getStorage_info () { return storage_info; }
-
- //TODO: is this a security concern? How would we get rid of the key once we're done with this value?
- public String getS3_key () { return s3_key; }
-
- public String getBucket_location () { return bucket_location; }
-
- public String getS3_accessId () { return s3_accessId; }
-
- public void setApplicationId (UUID appId) { applicationId = appId;}
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
deleted file mode 100644
index ed8a24e..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportJob.java
+++ /dev/null
@@ -1,64 +0,0 @@
-package org.usergrid.management.export;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.stereotype.Component;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.batch.job.OnlyOnceJob;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.persistence.entities.JobData;
-
-
-/**
- *
- *you could make an enum here, that contains the state info
- * look at scotts code and emulate that to see wha tyou can return in the json object
- */
-@Component("exportJob")
-public class ExportJob extends OnlyOnceJob {
- private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
-
- @Autowired
- ExportService exportService;
-
- public ExportJob() {
- logger.info("ExportJob created " + this);
- }
-
- @Override
- public void doJob(JobExecution jobExecution) throws Exception {
- logger.info( "execute ExportJob {}", jobExecution );
-
- ExportInfo config = null;
- //add check for null jobData.
- JobData jobData = jobExecution.getJobData();
- config = (ExportInfo) jobData.getProperty( "exportInfo" );
- if(jobData == null) {
- logger.error( "jobData cannot be null" );
- return;
- }
- else if (config == null) {
- logger.error( "Export information cannot be null" );
- return;
- }
- else {
- jobExecution.heartbeat();
- exportService.doExport( config,jobExecution );
- }
- logger.info( "executed ExportJob process completed" );
- }
-
- @Override
- protected long getDelay(JobExecution jobExecution) throws Exception {
- //return arbitrary number
- return 100;
- }
-
-
- @Autowired
- public void setExportService( final ExportService exportService ) {
- this.exportService = exportService;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java b/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
deleted file mode 100644
index 644aef6..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportService.java
+++ /dev/null
@@ -1,37 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.util.UUID;
-
-import org.usergrid.batch.JobExecution;
-import org.usergrid.management.ExportInfo;
-
-
-/**
- * Performs all functions related to exporting
- *
- */
-public interface ExportService {
-
- /**
- * Schedules the export to execute
- * @param config
- */
- UUID schedule(ExportInfo config) throws Exception;
-
-
- /**
- * Perform the export to the external resource
- * @param config
- */
- void doExport(ExportInfo config, JobExecution jobExecution) throws Exception;
-
- /**
- * Returns the current state of the service.
- * @return
- */
- String getState(UUID appId,UUID state) throws Exception;
-
- void setS3Export(S3Export s3Export);
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e1ab9df8/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
deleted file mode 100644
index 498828b..0000000
--- a/stack/services/src/main/java/org/usergrid/management/export/ExportServiceImpl.java
+++ /dev/null
@@ -1,454 +0,0 @@
-package org.usergrid.management.export;
-
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.util.DefaultPrettyPrinter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.usergrid.batch.JobExecution;
-import org.usergrid.batch.service.SchedulerService;
-import org.usergrid.management.ExportInfo;
-import org.usergrid.management.ManagementService;
-import org.usergrid.management.OrganizationInfo;
-import org.usergrid.persistence.ConnectionRef;
-import org.usergrid.persistence.Entity;
-import org.usergrid.persistence.EntityManager;
-import org.usergrid.persistence.EntityManagerFactory;
-import org.usergrid.persistence.PagingResultsIterator;
-import org.usergrid.persistence.Query;
-import org.usergrid.persistence.Results;
-import org.usergrid.persistence.entities.Export;
-import org.usergrid.persistence.entities.JobData;
-
-import com.google.common.collect.BiMap;
-
-
-/**
- *
- *Need to refactor out the mutliple orgs being take , need to factor out the multiple apps
- * it will just be the one app and the one org and all of it's collections.
- */
-public class ExportServiceImpl implements ExportService {
-
-
- private static final Logger logger = LoggerFactory.getLogger( ExportServiceImpl.class );
- public static final String EXPORT_ID = "exportId";
- public static final String EXPORT_JOB_NAME = "exportJob";
- //dependency injection
- private SchedulerService sch;
-
- //injected the Entity Manager Factory
- protected EntityManagerFactory emf;
-
- //inject Management Service to access Organization Data
- private ManagementService managementService;
-
- //Maximum amount of entities retrieved in a single go.
- public static final int MAX_ENTITY_FETCH = 100;
-
- //Amount of time that has passed before sending another heart beat in millis
- public static final int TIMESTAMP_DELTA = 5000;
-
- private JsonFactory jsonFactory = new JsonFactory();
-
- protected long startTime = System.currentTimeMillis();
-
- private S3Export s3Export;
-
- @Override
- public UUID schedule( final ExportInfo config ) throws Exception {
-
- EntityManager em = emf.getEntityManager( config.getApplicationId());
-
- Export export = new Export();
- export.setState( Export.State.PENDING );
-
- //validate that org exists,then app, then collection.
-
- String pathToBeParsed = config.getPath();
- //split the path so that you can verify that the organization and the app exist.
- String[] pathItems = pathToBeParsed.split( "/" );
-
-
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- }
-
- try {
- managementService.getApplicationInfo( pathItems[1] );
- }
- catch ( Exception e ) {
- logger.error( "Application doesn't exist" );
- }
-
-
- //TODO: parse path and make sure all the things you need actually exist. then throw
- // good error messages when not found.
-
- //write to the em
- export = em.create( export );
- export.setState( Export.State.PENDING );
- em.update( export );
-
- JobData jobData = new JobData();
- jobData.setProperty( "exportInfo", config );
- jobData.setProperty( EXPORT_ID, export.getUuid() );
-
- long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
-
- sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
-
- return export.getUuid();
-
- }
- /**
- * Query Entity Manager for specific Export Entity within application
- * @param appId,uuid
- * @return String
- * @throws Exception
- */
- @Override
- public String getState(final UUID appId, final UUID uuid) throws Exception {
-
- EntityManager rootEm = emf.getEntityManager( appId );
- Export export = rootEm.get( uuid, Export.class );
-
- if(export == null){
- return null;
- }
- return export.getState().toString();
- }
-
-
- @Override
- public void doExport( final ExportInfo config,final JobExecution jobExecution ) throws Exception {
-
- UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
- EntityManager em = emf.getEntityManager( config.getApplicationId());
- Export export = em.get( exportId, Export.class );
-
- String pathToBeParsed = config.getPath();
- String[] pathItems = pathToBeParsed.split( "/" );
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- return;
- }
-
- //update state and re-write the entity
- export.setState( Export.State.STARTED );
-
- em.update( export );
-
- Map<UUID, String> organizationGet = getOrgs(config);
- for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
- //needs to pass app name, and possibly collection to export
- exportApplicationsForOrg( organization, config, jobExecution );
- }
- export.setState( Export.State.COMPLETED );
- em.update( export );
- }
-
- private Map<UUID, String> getOrgs(ExportInfo exportInfo) throws Exception {
- // Loop through the organizations
- // TODO:this will come from the orgs in schedule when you do the validations. delete orgId
- UUID orgId = null;
-
- Map<UUID, String> organizationNames = null;
-
- if ( orgId == null ) {
- organizationNames = managementService.getOrganizations();
- }
-
- else {
- OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
-
- if ( info == null ) {
- logger.error( "Organization info is null!" );
- }
-
- organizationNames = new HashMap<UUID, String>();
- organizationNames.put( orgId, info.getName() );
- }
-
-
- return organizationNames;
- }
-
-
- public SchedulerService getSch() {
- return sch;
- }
-
-
- public void setSch( final SchedulerService sch ) {
- this.sch = sch;
- }
-
-
- public EntityManagerFactory getEmf() {
- return emf;
- }
-
-
- public void setEmf( final EntityManagerFactory emf ) {
- this.emf = emf;
- }
-
-
- public ManagementService getManagementService() {
-
- return managementService;
- }
-
- public void setManagementService( final ManagementService managementService ) {
- this.managementService = managementService;
- }
-
- //write test checking to see what happens if the input stream is closed or wrong.
- //TODO: make multipart streaming functional
- //currently only stores the collection in memory then flushes it.
- private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
- final JobExecution jobExecution ) throws Exception {
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
-
- logger.info( "" + organization );
-
- // Loop through the applications per organization
- BiMap<UUID, String> applications = managementService.getApplicationsForOrganization( organization.getKey() );
- for ( Map.Entry<UUID, String> application : applications.entrySet() ) {
-
- logger.info( application.getValue() + " : " + application.getKey() );
-
- String appFileName = prepareOutputFileName( "application", application.getValue() );
-
- JsonGenerator jg = getJsonGenerator( baos );
-
- EntityManager em = emf.getEntityManager( application.getKey() );
-
- jg.writeStartArray();
-
- Map<String, Object> metadata = em.getApplicationCollectionMetadata();
- long starting_time = System.currentTimeMillis();
-
- // Loop through the collections. This is the only way to loop
- // through the entities in the application (former namespace).
- //could support queries, just need to implement that in the rest endpoint.
- for ( String collectionName : metadata.keySet() ) {
- if(collectionName.equals( "exports" )) {
- continue;
- }
-
- Query query = new Query();
- query.setLimit( MAX_ENTITY_FETCH );
- query.setResultsLevel( Results.Level.ALL_PROPERTIES );
- Results entities = em.searchCollection( em.getApplicationRef(), collectionName, query );
-
- PagingResultsIterator itr = new PagingResultsIterator( entities );
-
- for( Object e: itr){
- starting_time = checkTimeDelta( starting_time, jobExecution );
- Entity entity = ( Entity ) e;
-
- jg.writeStartObject();
- jg.writeFieldName( "Metadata" );
- jg.writeObject( entity );
- saveCollectionMembers( jg, em, application.getValue(), entity );
- jg.writeEndObject();
- }
- }
-
- // Close writer and file for this application.
- jg.writeEndArray();
- jg.close();
- baos.flush();
- baos.close();
-
- InputStream is = new ByteArrayInputStream( baos.toByteArray() );
- s3Export.copyToS3( is, config , appFileName);
- }
- }
-
-
- public long checkTimeDelta( long startingTime, final JobExecution jobExecution ) {
-
- long cur_time = System.currentTimeMillis();
-
- if ( startingTime <= ( cur_time - TIMESTAMP_DELTA ) ) {
- jobExecution.heartbeat();
- return cur_time;
- }
- return startingTime;
- }
-
-
- /**
- * Serialize and save the collection members of this <code>entity</code>
- *
- * @param em Entity Manager
- * @param application Application name
- * @param entity entity
- */
- private void saveCollectionMembers( JsonGenerator jg, EntityManager em, String application, Entity entity )
- throws Exception {
-
- Set<String> collections = em.getCollections( entity );
- //jg.writeStartObject();
-
- // Only create entry for Entities that have collections
- if ( ( collections == null ) || collections.isEmpty() ) {
- return;
- }
-
-
- for ( String collectionName : collections ) {
-
- jg.writeFieldName( collectionName );
- jg.writeStartArray();
-
- //is 100000 an arbitary number?
- Results collectionMembers =
- em.getCollection( entity, collectionName, null, 100000, Results.Level.IDS, false );
-
- List<UUID> entityIds = collectionMembers.getIds();
-
- if ( ( entityIds != null ) && !entityIds.isEmpty() ) {
- for ( UUID childEntityUUID : entityIds ) {
- jg.writeObject( childEntityUUID.toString() );
- }
- }
-
- // End collection array.
- jg.writeEndArray();
- }
-
- // Write connections
- saveConnections( entity, em, jg );
-
- // Write dictionaries
- saveDictionaries( entity, em, jg );
-
- // End the object if it was Started
- //jg.writeEndObject();
- }
-
-
- /**
- * Persists the connection for this entity.
- */
- private void saveDictionaries( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
-
- jg.writeFieldName( "dictionaries" );
- jg.writeStartObject();
-
- Set<String> dictionaries = em.getDictionaries( entity );
- for ( String dictionary : dictionaries ) {
-
- Map<Object, Object> dict = em.getDictionaryAsMap( entity, dictionary );
-
- // nothing to do
- if ( dict.isEmpty() ) {
- continue;
- }
-
- jg.writeFieldName( dictionary );
-
- jg.writeStartObject();
-
- for ( Map.Entry<Object, Object> entry : dict.entrySet() ) {
- jg.writeFieldName( entry.getKey().toString() );
- jg.writeObject( entry.getValue() );
- }
-
- jg.writeEndObject();
- }
- jg.writeEndObject();
- }
-
-
- /**
- * Persists the connection for this entity.
- */
- private void saveConnections( Entity entity, EntityManager em, JsonGenerator jg ) throws Exception {
-
- jg.writeFieldName( "connections" );
- jg.writeStartObject();
-
- Set<String> connectionTypes = em.getConnectionTypes( entity );
- for ( String connectionType : connectionTypes ) {
-
- jg.writeFieldName( connectionType );
- jg.writeStartArray();
-
- Results results = em.getConnectedEntities( entity.getUuid(), connectionType, null, Results.Level.IDS );
- List<ConnectionRef> connections = results.getConnections();
-
- for ( ConnectionRef connectionRef : connections ) {
- jg.writeObject( connectionRef.getConnectedEntity().getUuid() );
- }
-
- jg.writeEndArray();
- }
- jg.writeEndObject();
- }
-
-
- protected JsonGenerator getJsonGenerator( ByteArrayOutputStream out ) throws IOException {
- //TODO:shouldn't the below be UTF-16?
- //PrintWriter out = new PrintWriter( outFile, "UTF-8" );
-
- JsonGenerator jg = jsonFactory.createJsonGenerator( out );
- jg.setPrettyPrinter( new DefaultPrettyPrinter() );
- jg.setCodec( new ObjectMapper() );
- return jg;
- }
-
-
- protected File createOutputFile( String type, String name ) {
- return new File( prepareOutputFileName( type, name ) );
- }
-
-
- /**
- * @param type just a label such us: organization, application.
- *
- * @return the file name concatenated with the type and the name of the collection
- */
- protected String prepareOutputFileName( String type, String name ) {
- StringBuilder str = new StringBuilder();
- str.append( name );
- str.append( "." );
- str.append( startTime );
- str.append( ".json" );
-
- String outputFileName = str.toString();
- //TODO:this is , i feel, bad practice so make sure to come back here and fix it up.
-
- return outputFileName;
- }
-
-
- @Autowired
- @Override
- public void setS3Export( S3Export s3Export ) { this.s3Export = s3Export; }
-}
[5/9] git commit: Removed JobInfo as it wasn’t used at all.
Posted by sn...@apache.org.
Removed JobInfo as it wasn’t used at all.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b1deced8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b1deced8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b1deced8
Branch: refs/pull/60/merge
Commit: b1deced83c63b7a200732cdde66a9a7fcd7c1b3d
Parents: e1ab9df
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 09:39:13 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 09:39:13 2014 -0800
----------------------------------------------------------------------
.../java/org/usergrid/management/JobInfo.java | 29 --------------------
1 file changed, 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b1deced8/stack/services/src/main/java/org/usergrid/management/JobInfo.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/usergrid/management/JobInfo.java b/stack/services/src/main/java/org/usergrid/management/JobInfo.java
deleted file mode 100644
index cc60b0e..0000000
--- a/stack/services/src/main/java/org/usergrid/management/JobInfo.java
+++ /dev/null
@@ -1,29 +0,0 @@
-package org.usergrid.management;
-
-import java.util.UUID;
-
-/**
- * Created by ApigeeCorporation on 1/31/14.
- */
-//TODO: Documentation on this class
-public class JobInfo {
- private UUID jobId;
-
- public enum StatusType{
- PENDING,STARTED,FAILED,COMPLETED,FAIL1, FAIL2
- }
- private StatusType jobStatusType;
-
- public JobInfo (UUID jobId, StatusType jobStatusType){
- jobId = this.jobId;
- jobStatusType = this.jobStatusType;
- }
-
- public StatusType getStatusType () {
- return jobStatusType;
- }
-
- public UUID getJobId () {
- return jobId;
- }
-}