You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/02/16 23:01:45 UTC
[38/50] incubator-usergrid git commit: Merge branch
'two-dot-o-import' into USERGRID-273
Merge branch 'two-dot-o-import' into USERGRID-273
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
stack/core/src/main/java/org/apache/usergrid/persistence/exceptions/OrganizationAlreadyExistsException.java
stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime1IT.java
stack/core/src/test/java/org/apache/usergrid/batch/job/TestJobListenerTest.java
stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
stack/core/src/test/resources/usergrid-custom-test.properties
stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
stack/pom.xml
stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java
stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c75b7a95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c75b7a95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c75b7a95
Branch: refs/heads/USERGRID-365
Commit: c75b7a952b5ea70ce253878a5d2e98aecc8bfb89
Parents: 84b048e c46d0d1
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Feb 16 11:05:13 2015 -0800
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Feb 16 11:05:13 2015 -0800
----------------------------------------------------------------------
portal/bower.json | 2 +-
portal/package.json | 2 +-
stack/awscluster/src/main/cql/update_locks.cql | 2 +-
.../awscluster/src/main/cql/update_usergrid.cql | 2 +-
.../main/cql/update_usergrid_applications.cql | 2 +-
.../main/dist/init_instance/init_rest_server.sh | 23 +-
.../main/dist/init_instance/install_collectd.sh | 350 +++++
.../dist/init_instance/install_elasticsearch.sh | 12 +-
.../main/dist/init_instance/update_keyspaces.sh | 9 +
.../src/main/groovy/NodeRegistry.groovy | 26 +
.../src/main/groovy/configure_usergrid.groovy | 29 +-
.../src/main/groovy/create_dashboard.groovy | 79 ++
.../src/main/groovy/wait_for_instances.groovy | 20 +-
stack/awscluster/ugcluster-cf.json | 25 +-
.../java/org/apache/usergrid/batch/Job.java | 3 +-
.../apache/usergrid/batch/JobExecutionImpl.java | 10 +
.../usergrid/corepersistence/CoreModule.java | 120 ++
.../corepersistence/CpEntityManager.java | 19 +-
.../corepersistence/CpRelationManager.java | 40 +-
.../usergrid/corepersistence/GuiceModule.java | 120 --
.../events/EntityDeletedHandler.java | 16 +-
.../events/EntityVersionCreatedHandler.java | 20 +-
.../events/EntityVersionDeletedHandler.java | 18 +-
.../corepersistence/util/CpEntityMapUtils.java | 55 +-
.../cassandra/EntityManagerImpl.java | 1 -
.../persistence/entities/FailedImport.java | 64 +
.../entities/FailedImportConnection.java | 41 +
.../entities/FailedImportEntity.java | 44 +
.../persistence/entities/FileImport.java | 211 +++
.../usergrid/persistence/entities/Import.java | 120 ++
.../exceptions/PersistenceException.java | 2 +-
.../org/apache/usergrid/utils/JsonUtils.java | 6 +-
.../corepersistence/StaleIndexCleanupTest.java | 35 +-
.../persistence/EntityConnectionsIT.java | 101 ++
.../index/impl/EsEntityIndexImpl.java | 6 +-
.../usergrid/persistence/index/query/Query.java | 47 +-
.../persistence/queue/DefaultQueueManager.java | 6 +-
.../persistence/queue/QueueMessage.java | 10 +-
.../queue/impl/QueueScopeFactoryImpl.java | 1 +
.../queue/impl/SQSQueueManagerImpl.java | 56 +-
.../queue/impl/UsergridAwsCredentials.java | 75 +
.../impl/UsergridAwsCredentialsProvider.java | 59 +
.../datagenerators/FeederGenerator.scala | 22 +-
.../usergrid/scenarios/EntityScenarios.scala | 12 +-
.../PostCustomEntitySimulation.scala | 12 +-
.../PushNotificationTargetUserSimulation.scala | 5 +-
stack/pom.xml | 10 +-
.../usergrid/rest/AbstractContextResource.java | 40 +-
.../org/apache/usergrid/rest/ApiResponse.java | 16 +
.../usergrid/rest/JobServiceBoostrap.java | 72 +
.../rest/applications/ServiceResource.java | 28 +-
.../exceptions/NullArgumentExceptionMapper.java | 42 +
.../usergrid/rest/filters/MeteringFilter.java | 6 +-
.../organizations/OrganizationResource.java | 73 +-
.../applications/ApplicationResource.java | 142 +-
.../imports/FileErrorsResource.java | 143 ++
.../imports/FileIncludesResource.java | 158 +++
.../applications/imports/ImportsResource.java | 220 +++
.../main/resources/usergrid-rest-context.xml | 6 +-
.../rest/management/ExportResourceIT.java | 11 +-
.../rest/management/ImportResourceIT.java | 769 ++++++++++
.../rest/management/OrganizationsIT.java | 2 +-
.../test/resource2point0/AbstractRestIT.java | 7 +
.../endpoints/mgmt/ApplicationResource.java | 23 +
.../endpoints/mgmt/OrganizationResource.java | 9 +-
.../rest/test/resource2point0/model/Entity.java | 4 +
.../resources/corepersistence-UNIT.properties | 2 +
.../resources/testImportCorrect.testCol.1.json | 18 +
...testImportInvalidJson.testApplication.3.json | 153 ++
stack/services/pom.xml | 3 +-
.../cassandra/ManagementServiceImpl.java | 7 +-
.../usergrid/management/export/ExportJob.java | 6 +-
.../management/export/ExportServiceImpl.java | 43 +-
.../usergrid/management/export/S3Export.java | 7 -
.../management/export/S3ExportImpl.java | 62 +-
.../management/importer/FileImportJob.java | 123 ++
.../management/importer/FileImportTracker.java | 319 +++++
.../usergrid/management/importer/ImportJob.java | 114 ++
.../management/importer/ImportService.java | 133 ++
.../management/importer/ImportServiceImpl.java | 1324 ++++++++++++++++++
.../usergrid/management/importer/S3Import.java | 37 +
.../management/importer/S3ImportImpl.java | 129 ++
.../usergrid/services/guice/ServiceModule.java | 68 +
.../notifications/NotificationsService.java | 2 +-
.../notifications/ProviderAdapterFactory.java | 1 -
.../services/notifications/QueueListener.java | 5 +
.../services/queues/ImportQueueListener.java | 90 ++
.../services/queues/ImportQueueManager.java | 65 +
.../services/queues/ImportQueueMessage.java | 79 ++
.../usergrid/services/queues/QueueListener.java | 252 ++++
.../resources/usergrid-services-context.xml | 218 +--
.../org/apache/usergrid/ServiceITSetup.java | 7 +-
.../org/apache/usergrid/ServiceITSetupImpl.java | 12 +
.../management/cassandra/ExportServiceIT.java | 1041 --------------
.../management/cassandra/MockS3ExportImpl.java | 56 -
.../management/export/ExportServiceIT.java | 1073 ++++++++++++++
.../management/export/MockS3ExportImpl.java | 56 +
.../importer/FileImportTrackerTest.java | 410 ++++++
.../management/importer/ImportCollectionIT.java | 647 +++++++++
.../importer/ImportConnectionsTest.java | 183 +++
.../management/importer/ImportServiceIT.java | 672 +++++++++
.../management/importer/MockS3ImportImpl.java | 42 +
.../usergrid/management/importer/S3Upload.java | 105 ++
.../src/test/resources/log4j.properties | 10 +-
.../resources/testImport.testApplication.2.json | 56 +
.../resources/testImport.testCollection.1.json | 13 +
...testImportInvalidJson.testApplication.3.json | 139 ++
.../resources/testimport-bad-connection.json | 18 +
.../src/test/resources/testimport-bad-json.json | 67 +
.../src/test/resources/testimport-qtmagics.json | 66 +
.../resources/testimport-with-connections.json | 155 ++
.../resources/usergrid-custom-test.properties | 5 +-
.../org/apache/usergrid/tools/IndexRebuild.java | 11 +-
113 files changed, 10038 insertions(+), 1787 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 0000000,5785739..699dfec
mode 000000,100644..100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@@ -1,0 -1,94 +1,120 @@@
+ /*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.usergrid.corepersistence;
+
+ import com.google.inject.AbstractModule;
++import com.google.inject.Provider;
+ import com.google.inject.multibindings.Multibinder;
+
+ import org.apache.usergrid.corepersistence.migration.EntityDataMigration;
+ import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
+ import org.apache.usergrid.corepersistence.migration.GraphShardVersionMigration;
+ import org.apache.usergrid.corepersistence.events.EntityDeletedHandler;
+ import org.apache.usergrid.corepersistence.events.EntityVersionCreatedHandler;
+ import org.apache.usergrid.corepersistence.events.EntityVersionDeletedHandler;
+ import org.apache.usergrid.persistence.EntityManagerFactory;
+ import org.apache.usergrid.persistence.collection.event.EntityDeleted;
+ import org.apache.usergrid.persistence.collection.event.EntityVersionCreated;
+ import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
+ import org.apache.usergrid.persistence.collection.guice.CollectionModule;
+ import org.apache.usergrid.persistence.core.guice.CommonModule;
+ import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+ import org.apache.usergrid.persistence.graph.guice.GraphModule;
+ import org.apache.usergrid.persistence.index.guice.IndexModule;
+ import org.apache.usergrid.persistence.map.guice.MapModule;
+ import org.apache.usergrid.persistence.queue.guice.QueueModule;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
++import org.springframework.context.ApplicationContext;
+
+
+ /**
+ * Guice Module that encapsulates Core Persistence.
+ */
-public class CoreModule extends AbstractModule {
++public class GuiceModule extends AbstractModule {
+
- private static final Logger logger = LoggerFactory.getLogger(CoreModule.class);
-
- private EntityManagerFactory emf;
++ /**
++ * TODO this is a circular dependency, and should be refactored
++ */
++ private LazyEntityManagerFactoryProvider lazyEntityManagerFactoryProvider;
+
+ public static final String EVENTS_DISABLED = "corepersistence.events.disabled";
+
- public CoreModule( EntityManagerFactory emf ) {
- this.emf = emf;
++
++
++ public GuiceModule( final ApplicationContext context ) {
++ this.lazyEntityManagerFactoryProvider = new LazyEntityManagerFactoryProvider( context );
+ }
+
+ @Override
+ protected void configure() {
+
- if ( emf != null ) {
- bind( EntityManagerFactory.class ).toInstance( emf );
- }
+
- install(new CommonModule());
++ //See TODO, this is fugly
++ bind(EntityManagerFactory.class).toProvider( lazyEntityManagerFactoryProvider );
++
++ install( new CommonModule());
+ install(new CollectionModule());
+ install(new GraphModule());
+ install(new IndexModule());
+ install(new MapModule());
+ install(new QueueModule());
+
+ bind(ManagerCache.class).to( CpManagerCache.class );
+
+ Multibinder<DataMigration> dataMigrationMultibinder =
+ Multibinder.newSetBinder( binder(), DataMigration.class );
+ dataMigrationMultibinder.addBinding().to( EntityTypeMappingMigration.class );
+ dataMigrationMultibinder.addBinding().to( GraphShardVersionMigration.class );
+ dataMigrationMultibinder.addBinding().to( EntityDataMigration.class );
+
+ Multibinder<EntityDeleted> entityBinder =
+ Multibinder.newSetBinder(binder(), EntityDeleted.class);
+ entityBinder.addBinding().to(EntityDeletedHandler.class);
+
+ Multibinder<EntityVersionDeleted> versionBinder =
+ Multibinder.newSetBinder(binder(), EntityVersionDeleted.class);
+ versionBinder.addBinding().to(EntityVersionDeletedHandler.class);
+
+ Multibinder<EntityVersionCreated> versionCreatedMultibinder =
+ Multibinder.newSetBinder( binder(), EntityVersionCreated.class );
+ versionCreatedMultibinder.addBinding().to(EntityVersionCreatedHandler.class);
+
+
+ }
+
++
++ /**
++ * TODO, this is a hack workaround due to the guice/spring EMF circular dependency
++ * Once the entity managers have been refactored and moved into guice, remove this dependency.
++ *
++ */
++ public static class LazyEntityManagerFactoryProvider implements Provider<EntityManagerFactory>{
++
++ private final ApplicationContext context;
++
++
++ public LazyEntityManagerFactoryProvider( final ApplicationContext context ) {this.context = context;}
++
++
++
++ @Override
++ public EntityManagerFactory get() {
++ return this.context.getBean( EntityManagerFactory.class );
++ }
++ }
++
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 0f0d401,9c7eeb9..0a22845
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -2750,25 -2755,8 +2755,13 @@@ public class CpEntityManager implement
ei.initializeIndex();
}
+ public void deleteIndex(){
+ EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ ei.deleteIndex();
+ }
+
- public static org.apache.usergrid.persistence.model.entity.Entity entityToCpEntity( Entity entity, UUID importId ) {
-
- UUID uuid = importId != null ? importId : entity.getUuid();
-
- org.apache.usergrid.persistence.model.entity.Entity cpEntity =
- new org.apache.usergrid.persistence.model.entity.Entity( new SimpleId( uuid, entity.getType() ) );
-
- cpEntity = CpEntityMapUtils.fromMap( cpEntity, entity.getProperties(), entity.getType(), true );
- cpEntity = CpEntityMapUtils.fromMap( cpEntity, entity.getDynamicProperties(), entity.getType(), true );
-
- return cpEntity;
- }
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --cc stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 267bd0b,4fd55c8..d356734
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@@ -242,10 -237,10 +240,10 @@@ public class StaleIndexCleanupTest exte
/**
- * Test that the EntityDeleteImpl cleans up stale indexes on delete.
- * Ensures that when an entity is deleted its old indexes are cleared from ElasticSearch.
+ * Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
+ * entity is deleted its old indexes are cleared from ElasticSearch.
*/
- @Test(timeout=10000)
+ @Test(timeout=30000)
public void testCleanupOnDelete() throws Exception {
logger.info("Started testStaleIndexCleanup()");
@@@ -322,10 -317,10 +320,10 @@@
/**
- * Test that the EntityDeleteImpl cleans up stale indexes on update.
- * Ensures that when an entity is updated its old indexes are cleared from ElasticSearch.
+ * Test that the EntityDeleteImpl cleans up stale indexes on update. Ensures that when an
+ * entity is updated its old indexes are cleared from ElasticSearch.
*/
- @Test(timeout=10000)
+ @Test(timeout=30000)
public void testCleanupOnUpdate() throws Exception {
logger.info( "Started testCleanupOnUpdate()" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/core/src/test/java/org/apache/usergrid/persistence/EntityConnectionsIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/rest/src/test/java/org/apache/usergrid/rest/management/ExportResourceIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java
index 1b505c3,04a1b1f..78fe11e
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/AbstractRestIT.java
@@@ -21,16 -21,20 +21,19 @@@ import java.net.URI
import java.net.URLClassLoader;
import java.util.Arrays;
-import com.fasterxml.jackson.databind.ObjectMapper;
+import org.junit.Rule;
+
+import org.apache.usergrid.rest.ITSetup;
import org.apache.usergrid.rest.test.resource2point0.endpoints.ApplicationsResource;
import org.apache.usergrid.rest.test.resource2point0.endpoints.OrganizationResource;
+ import org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt.ManagementResource;
+ import org.apache.usergrid.rest.test.resource2point0.state.ClientContext;
+ import org.apache.usergrid.rest.test.resource2point0.model.Entity;
import org.apache.usergrid.rest.test.resource2point0.model.Token;
-import org.junit.ClassRule;
-import org.junit.Rule;
-
-import org.apache.usergrid.rest.ITSetup;
-import org.apache.usergrid.rest.RestITSuite;
+import org.apache.usergrid.rest.test.resource2point0.state.ClientContext;
import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.sun.jersey.api.client.UniformInterfaceException;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
----------------------------------------------------------------------
diff --cc stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
index a820a7b,79982c0..5975d4e
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/resource2point0/endpoints/mgmt/ApplicationResource.java
@@@ -20,14 -20,16 +20,15 @@@
package org.apache.usergrid.rest.test.resource2point0.endpoints.mgmt;
-import com.sun.jersey.api.client.WebResource;
-import org.apache.usergrid.batch.service.App;
+
+import javax.ws.rs.core.MediaType;
+
import org.apache.usergrid.rest.test.resource2point0.endpoints.NamedResource;
import org.apache.usergrid.rest.test.resource2point0.endpoints.UrlResource;
+import org.apache.usergrid.rest.test.resource2point0.model.Application;
+ import org.apache.usergrid.rest.test.resource2point0.model.*;
import org.apache.usergrid.rest.test.resource2point0.state.ClientContext;
-import javax.ws.rs.core.MediaType;
-import java.util.Map;
-
/**
* Classy class class.
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/services/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/services/src/main/java/org/apache/usergrid/services/notifications/NotificationsService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/services/src/main/java/org/apache/usergrid/services/notifications/QueueListener.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
index 93b3fb9,cd8b691..c466080
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSetupImpl.java
@@@ -29,7 -21,9 +29,8 @@@ import org.apache.shiro.SecurityUtils
import org.apache.usergrid.management.ApplicationCreator;
import org.apache.usergrid.management.ManagementService;
import org.apache.usergrid.management.export.ExportService;
+ import org.apache.usergrid.management.importer.ImportService;
import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
import org.apache.usergrid.security.providers.SignInProviderFactory;
import org.apache.usergrid.security.tokens.TokenService;
import org.apache.usergrid.services.ServiceManagerFactory;
@@@ -46,20 -47,11 +54,21 @@@ public class ServiceITSetupImpl extend
private SignInProviderFactory providerFactory;
private Properties properties;
private ExportService exportService;
+ private ImportService importService;
- public ServiceITSetupImpl( CassandraResource cassandraResource, ElasticSearchResource elasticSearchResource ) {
- super( cassandraResource, elasticSearchResource );
+ public ServiceITSetupImpl() {
+ super();
+ managementService = springResource.getBean( ManagementService.class );
+ applicationCreator = springResource.getBean( ApplicationCreator.class );
+ tokenService = springResource.getBean( TokenService.class );
+ providerFactory = springResource.getBean( SignInProviderFactory.class );
+ properties = springResource.getBean( "properties", Properties.class );
+ smf = springResource.getBean( ServiceManagerFactory.class );
+ exportService = springResource.getBean( ExportService.class );
+
+ //set our security manager for shiro
+ SecurityUtils.setSecurityManager(springResource.getBean( org.apache.shiro.mgt.SecurityManager.class ));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c75b7a95/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
----------------------------------------------------------------------
diff --cc stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
index 0000000,d89ae59..f329739
mode 000000,100644..100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/export/ExportServiceIT.java
@@@ -1,0 -1,1075 +1,1073 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ package org.apache.usergrid.management.export;
+
+
+ import java.io.File;
+ import java.io.FileReader;
+ import java.util.*;
+
+ import com.google.common.util.concurrent.Service;
+ import org.apache.commons.lang.RandomStringUtils;
+ import org.apache.commons.lang3.StringUtils;
+ import org.apache.usergrid.batch.service.JobSchedulerService;
+ import org.jclouds.ContextBuilder;
+ import org.jclouds.blobstore.BlobStore;
+ import org.jclouds.blobstore.BlobStoreContext;
+ import org.jclouds.blobstore.domain.Blob;
+ import org.jclouds.http.config.JavaUrlHttpCommandExecutorServiceModule;
+ import org.jclouds.logging.log4j.config.Log4JLoggingModule;
+ import org.jclouds.netty.config.NettyPayloadModule;
+ import org.json.simple.JSONObject;
+ import org.json.simple.parser.JSONParser;
+ import org.junit.*;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import org.apache.usergrid.NewOrgAppAdminRule;
+ import org.apache.usergrid.ServiceITSetup;
+ import org.apache.usergrid.ServiceITSetupImpl;
+ import org.apache.usergrid.batch.JobExecution;
-import org.apache.usergrid.cassandra.CassandraResource;
++import org.apache.usergrid.cassandra.SpringResource;
+ import org.apache.usergrid.cassandra.ClearShiroSubject;
++
+ import org.apache.usergrid.management.ApplicationInfo;
+ import org.apache.usergrid.management.OrganizationInfo;
+ import org.apache.usergrid.management.UserInfo;
+ import org.apache.usergrid.persistence.Entity;
+ import org.apache.usergrid.persistence.EntityManager;
+ import org.apache.usergrid.persistence.SimpleEntityRef;
+ import org.apache.usergrid.persistence.entities.JobData;
+ import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
+
+ import com.amazonaws.SDKGlobalConfiguration;
+ import com.google.common.collect.ImmutableSet;
+ import com.google.inject.Module;
+
+ import static org.apache.usergrid.TestHelper.newUUIDString;
+ import static org.apache.usergrid.TestHelper.uniqueApp;
+ import static org.apache.usergrid.TestHelper.uniqueOrg;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertNotNull;
+ import static org.junit.Assert.assertTrue;
+ import static org.mockito.Mockito.mock;
+ import static org.mockito.Mockito.when;
+
+
++/**
++ *
++ *
++ */
+ public class ExportServiceIT {
+
+ private static final Logger logger = LoggerFactory.getLogger( ExportServiceIT.class );
+
- @ClassRule
- public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
-
- @ClassRule
- public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
-
+
+ @ClassRule
- public static final ServiceITSetup setup =
- new ServiceITSetupImpl( cassandraResource, elasticSearchResource );
++ public static final ServiceITSetup setup = new ServiceITSetupImpl( );
+
+ @Rule
+ public ClearShiroSubject clearShiroSubject = new ClearShiroSubject();
+
+ @Rule
+ public NewOrgAppAdminRule newOrgAppAdminRule = new NewOrgAppAdminRule( setup );
+
+ // app-level data generated only once
+ private UserInfo adminUser;
+ private OrganizationInfo organization;
+ private UUID applicationId;
+
+ private static String bucketPrefix;
+
+ private String bucketName;
+
+ @Before
+ public void setup() throws Exception {
+ logger.info("in setup");
+
+ // start the scheduler after we're all set up
- JobSchedulerService jobScheduler = cassandraResource.getBean( JobSchedulerService.class );
++ JobSchedulerService jobScheduler = ConcurrentSasdfasdf.getBean( JobSchedulerService.class );
+ if ( jobScheduler.state() != Service.State.RUNNING ) {
+ jobScheduler.startAndWait();
+ }
+
+ adminUser = newOrgAppAdminRule.getAdminInfo();
+ organization = newOrgAppAdminRule.getOrganizationInfo();
+ applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
+
+ setup.getEmf().refreshIndex();
+ }
+
+
+ @Before
+ public void before() {
+
+ boolean configured =
+ !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR))
+ && !StringUtils.isEmpty(System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR))
+ && !StringUtils.isEmpty(System.getProperty("bucketName"));
+
+ if ( !configured ) {
+ logger.warn("Skipping test because {}, {} and bucketName not " +
+ "specified as system properties, e.g. in your Maven settings.xml file.",
+ new Object[] {
+ SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
+ SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR
+ });
+ }
+
+ Assume.assumeTrue( configured );
+
+ adminUser = newOrgAppAdminRule.getAdminInfo();
+ organization = newOrgAppAdminRule.getOrganizationInfo();
+ applicationId = newOrgAppAdminRule.getApplicationInfo().getId();
+
+ bucketPrefix = System.getProperty( "bucketName" );
+ bucketName = bucketPrefix + RandomStringUtils.randomAlphanumeric(10).toLowerCase();
+ }
+
+
+ //Tests to make sure we can call the job with mock data and it runs.
+ @Ignore("Connections won't save when run with maven, but on local builds it will.")
+ public void testConnectionsOnCollectionExport() throws Exception {
+
+ File f = null;
+ int indexCon = 0;
+
+ try {
+ f = new File( "testFileConnections.json" );
+ }
+ catch ( Exception e ) {
+ // consumed because this checks to see if the file exists.
+ // If it doesn't then don't do anything and carry on.
+ }
+ f.deleteOnExit();
+
+ S3Export s3Export = new MockS3ExportImpl("testFileConnections.json" );
+
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+ payload.put( "collectionName", "users" );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[2];
+ //creates entities
+ for ( int i = 0; i < 2; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "meatIsGreat" + i );
+ userProperties.put( "email", "grey" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
+
+ entity[i] = em.create( "users", userProperties );
+ }
+ //creates connections
+ em.createConnection(
+ em.get( new SimpleEntityRef( "user", entity[0].getUuid()) ), "Vibrations",
+ em.get( new SimpleEntityRef( "user", entity[1].getUuid()) ) );
+ em.createConnection(
+ em.get( new SimpleEntityRef( "user", entity[1].getUuid()) ), "Vibrations",
+ em.get( new SimpleEntityRef( "user", entity[0].getUuid()) ) );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ //assertEquals(2, a.size() );
+
+ for ( indexCon = 0; indexCon < a.size(); indexCon++ ) {
+ JSONObject jObj = ( JSONObject ) a.get( indexCon );
+ JSONObject data = ( JSONObject ) jObj.get( "Metadata" );
+ String uuid = ( String ) data.get( "uuid" );
+ if ( entity[0].getUuid().toString().equals( uuid ) ) {
+ break;
+ }
+ }
+
+ org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( indexCon );
+ org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
+
+ assertNotNull( objConnections );
+
+ org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
+
+ assertNotNull( objVibrations );
+
+
+ }
+
+
+ @Test //Connections won't save when run with maven, but on local builds it will.
+ public void testConnectionsOnApplicationEndpoint() throws Exception {
+
+ File f = null;
+
+ try {
+ f = new File( "testConnectionsOnApplicationEndpoint.json" );
+ }
+ catch ( Exception e ) {
+ // consumed because this checks to see if the file exists.
+ // If it doesn't then don't do anything and carry on.
+ }
+
+ String fileName = "testConnectionsOnApplicationEndpoint.json";
+
+ S3Export s3Export = new MockS3ExportImpl( "testConnectionsOnApplicationEndpoint.json" );
+
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+ // intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[2];
+
+ // creates entities
+ for ( int i = 0; i < 2; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );//String.format( "test%i@anuff.com", i ) );
+
+ entity[i] = em.create( "users", userProperties );
+ }
+ em.refreshIndex();
+ //creates connections
+ em.createConnection(
+ em.get( new SimpleEntityRef( "user", entity[0].getUuid())), "Vibrations",
+ em.get( new SimpleEntityRef( "user", entity[1].getUuid())) );
+ em.createConnection(
+ em.get( new SimpleEntityRef( "user", entity[1].getUuid())), "Vibrations",
+ em.get( new SimpleEntityRef( "user", entity[0].getUuid())) );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ //create and initialize jobData returned in JobExecution.
+ JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ int indexApp = 0;
+
+ for ( indexApp = 0; indexApp < a.size(); indexApp++ ) {
+ JSONObject jObj = ( JSONObject ) a.get( indexApp );
+ JSONObject data = ( JSONObject ) jObj.get( "Metadata" );
+ String uuid = ( String ) data.get( "uuid" );
+ if ( entity[0].getUuid().toString().equals( uuid ) ) {
+ break;
+ }
+ }
+
+ assertTrue( "Uuid was not found in exported files. ", indexApp < a.size() );
+
+
+ org.json.simple.JSONObject objEnt = ( org.json.simple.JSONObject ) a.get( indexApp );
+ org.json.simple.JSONObject objConnections = ( org.json.simple.JSONObject ) objEnt.get( "connections" );
+
+ assertNotNull( objConnections );
+
+ org.json.simple.JSONArray objVibrations = ( org.json.simple.JSONArray ) objConnections.get( "Vibrations" );
+
+ assertNotNull( objVibrations );
+
+ f.deleteOnExit();
+ }
+
+ @Test
+ public void testExportOneOrgCollectionEndpoint() throws Exception {
+
+ File f = null;
+
+
+ try {
+ f = new File( "exportOneOrg.json" );
+ }
+ catch ( Exception e ) {
+ //consumed because this checks to see if the file exists.
+ // If it doesn't then don't do anything and carry on.
+ }
+
+ //create another org to ensure we don't export it
+ newOrgAppAdminRule.createOwnerAndOrganization(
+ "noExport"+newUUIDString(),
+ "junkUserName"+newUUIDString(),
+ "junkRealName"+newUUIDString(),
+ newUUIDString()+"ugExport@usergrid.com",
+ "123456789" );
+
+ S3Export s3Export = new MockS3ExportImpl("exportOneOrg.json");
+ // s3Export.setFilename( "exportOneOrg.json" );
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+ payload.put( "collectionName", "roles" );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
+
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+
+ assertEquals( 3, a.size() );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject entity = ( org.json.simple.JSONObject ) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) entity.get( "Metadata" );
+ String entityName = ( String ) entityData.get( "name" );
+ // assertNotEquals( "NotEqual","junkRealName",entityName );
+ assertFalse( "junkRealName".equals( entityName ) );
+ }
+ f.deleteOnExit();
+ }
+
+
+ //
+ //creation of files doesn't always delete itself
+ @Test
+ public void testExportOneAppOnCollectionEndpoint() throws Exception {
+
+ final String orgName = uniqueOrg();
+ final String appName = uniqueApp();
+
+
+ File f = null;
+
+ try {
+ f = new File( "exportOneApp.json" );
+ }
+ catch ( Exception e ) {
+ // consumed because this checks to see if the file exists.
+ // If it doesn't, don't do anything and carry on.
+ }
+ f.deleteOnExit();
+
+
+ UUID appId = setup.getEmf().createApplication( orgName, appName );
+
+
+ EntityManager em = setup.getEmf().getEntityManager( appId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[1];
+ //creates entities
+ for ( int i = 0; i < 1; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "junkRealName" );
+ userProperties.put( "email", "test" + i + "@anuff.com" );
+ entity[i] = em.create( "user", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl("exportOneApp.json");
+ //s3Export.setFilename( "exportOneApp.json" );
+ ExportService exportService = setup.getExportService();
+
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ exportService.doExport( jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+
+ //assertEquals( 3 , a.size() );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
+ String entityName = ( String ) entityData.get( "name" );
+ assertFalse( "junkRealName".equals( entityName ) );
+ }
+ }
+
+
+ @Test
+ public void testExportOneAppOnApplicationEndpointWQuery() throws Exception {
+
+ File f = null;
+ try {
+ f = new File( "exportOneAppWQuery.json" );
+ }
+ catch ( Exception e ) {
+ // consumed because this checks to see if the file exists.
+ // If it doesn't, don't do anything and carry on.
+ }
+ f.deleteOnExit();
+
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[1];
+ //creates entities
+ for ( int i = 0; i < 1; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "name", "me" );
+ userProperties.put( "username", "junkRealName" );
+ userProperties.put( "email", "burp" + i + "@anuff.com" );
+ entity[i] = em.create( "users", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl("exportOneAppWQuery.json" );
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "query", "select * where username = 'junkRealName'" );
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ em.refreshIndex();
+
+ exportService.doExport( jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+ assertEquals( 1, a.size() );
+ for ( int i = 0; i < a.size(); i++ ) {
+ org.json.simple.JSONObject data = ( org.json.simple.JSONObject ) a.get( i );
+ org.json.simple.JSONObject entityData = ( JSONObject ) data.get( "Metadata" );
+ String entityName = ( String ) entityData.get( "name" );
+ assertFalse( "junkRealName".equals( entityName ) );
+ }
+ }
+
+
+ @Test
+ public void testExportOneCollection() throws Exception {
+
+ File f = null;
+ int entitiesToCreate = 5;
+
+ try {
+ f = new File( "exportOneCollection.json" );
+ }
+ catch ( Exception e ) {
+ // consumed because this checks to see if the file exists.
+ // If it doesn't, don't do anything and carry on.
+ }
+
+ f.deleteOnExit();
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ // em.createApplicationCollection( "qtsMagics" );
+ // intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[entitiesToCreate];
+ //creates entities
+ for ( int i = 0; i < entitiesToCreate; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );
+ entity[i] = em.create( "qtsMagics", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl("exportOneCollection.json" );
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+ payload.put( "collectionName", "qtsMagics" );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ JobData jobData = jobDataCreator(payload,exportUUID,s3Export);
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ em.refreshIndex();
+
+ exportService.doExport( jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+
+ assertEquals( entitiesToCreate, a.size() );
+ }
+
+
+ @Test
+ public void testExportOneCollectionWQuery() throws Exception {
+
+ File f = null;
+ int entitiesToCreate = 5;
+
+ try {
+ f = new File( "exportOneCollectionWQuery.json" );
+ }
+ catch ( Exception e ) {
+ // consumed because this checks to see if the file exists.
+ // If it doesn't, don't do anything and carry on.
+ }
+ f.deleteOnExit();
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+ em.createApplicationCollection( "baconators" );
+ em.refreshIndex();
+
+ //initialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[entitiesToCreate];
+
+ // creates entities
+ for ( int i = 0; i < entitiesToCreate; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "billybob" + i );
+ userProperties.put( "email", "test" + i + "@anuff.com" );
+ entity[i] = em.create( "baconators", userProperties );
+ }
+
+ S3Export s3Export = new MockS3ExportImpl("exportOneCollectionWQuery.json");
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "query", "select * where username contains 'billybob0'" );
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+ payload.put( "collectionName", "baconators" );
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
+
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+
+ em.refreshIndex();
+
+ exportService.doExport( jobExecution );
+
+ JSONParser parser = new JSONParser();
+
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray ) parser.parse( new FileReader( f ) );
+
+ // only one entity should match the query.
+ assertEquals( 1, a.size() );
+ }
+
+
+ @Test
+ public void testExportOneOrganization() throws Exception {
+
+ // create a bunch of organizations, each with applications and collections of entities
+
+ int maxOrgs = 3;
+ int maxApps = 3;
+ int maxEntities = 20;
+
+ List<ApplicationInfo> appsMade = new ArrayList<>();
+ List<OrganizationInfo> orgsMade = new ArrayList<>();
+
+ for ( int orgIndex = 0; orgIndex < maxOrgs; orgIndex++ ) {
+
+
+ String orgName = "org_" + RandomStringUtils.randomAlphanumeric(10);
+ OrganizationInfo orgMade = setup.getMgmtSvc().createOrganization( orgName, adminUser, true );
+ orgsMade.add( orgMade );
+ logger.debug("Created org {}", orgName);
+
+ for ( int appIndex = 0; appIndex < maxApps; appIndex++ ) {
+
+ String appName = "app_" + RandomStringUtils.randomAlphanumeric(10);
+ ApplicationInfo appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), appName );
+ appsMade.add( appMade );
+ logger.debug("Created app {}", appName);
+
+ for (int entityIndex = 0; entityIndex < maxEntities; entityIndex++) {
+
+ EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );
+ appEm.create( appName + "_type", new HashMap<String, Object>() {{
+ put("property1", "value1");
+ put("property2", "value2");
+ }});
+ }
+ }
+ }
+
+ // export one of the organizations only, using mock S3 export that writes to local disk
+
+ String exportFileName = "exportOneOrganization.json";
+ S3Export s3Export = new MockS3ExportImpl( exportFileName );
+
+ HashMap<String, Object> payload = payloadBuilder(appsMade.get(0).getName());
+ payload.put("organizationId", orgsMade.get(0).getUuid() );
+ payload.put( "applicationId", appsMade.get(0).getId() );
+
+ ExportService exportService = setup.getExportService();
+ UUID exportUUID = exportService.schedule( payload );
+
+ JobData jobData = jobDataCreator( payload, exportUUID, s3Export );
+ JobExecution jobExecution = mock( JobExecution.class );
+ when( jobExecution.getJobData() ).thenReturn(jobData);
+
+ exportService.doExport( jobExecution );
+
+ // finally, we check that file was created and contains the right number of entities in the array
+
+ File exportedFile = new File( exportFileName );
+ exportedFile.deleteOnExit();
+
+ JSONParser parser = new JSONParser();
+ org.json.simple.JSONArray a = ( org.json.simple.JSONArray )
+ parser.parse( new FileReader( exportedFile ) );
+
+ assertEquals(23, a.size());
+ }
+
+
+ @Test
+ public void testExportDoJob() throws Exception {
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+
+ JobData jobData = new JobData();
+ jobData.setProperty( "jobName", "exportJob" );
+
+ // this needs to be populated with properties of export info
+ jobData.setProperty( "exportInfo", payload );
+
+ JobExecution jobExecution = mock( JobExecution.class );
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+ when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
+
+ ExportJob job = new ExportJob();
+ ExportService eS = mock( ExportService.class );
+ job.setExportService( eS );
+ try {
+ job.doJob( jobExecution );
+ }
+ catch ( Exception e ) {
+ logger.error("Error doing job", e);
+ assert ( false );
+ }
+ assert ( true );
+ }
+
+ //tests that with empty job data, the export still runs.
+ @Test
+ public void testExportEmptyJobData() throws Exception {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock( JobExecution.class );
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+ when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ //setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
+ }
+
+
+ @Test
+ public void testNullJobExecution() {
+
+ JobData jobData = new JobData();
+
+ JobExecution jobExecution = mock( JobExecution.class );
+
+ when( jobExecution.getJobData() ).thenReturn( jobData );
+ when( jobExecution.getJobId() ).thenReturn( UUID.randomUUID() );
+
+ ExportJob job = new ExportJob();
+ S3Export s3Export = mock( S3Export.class );
+ // setup.getExportService().setS3Export( s3Export );
+ job.setExportService( setup.getExportService() );
+ try {
+ job.doJob( jobExecution );
+ }
+ catch ( Exception e ) {
+ assert ( false );
+ }
+ assert ( true );
+ }
+
+
+ @Test
+ @Ignore // TODO: fix this test...
+ public void testIntegration100EntitiesOn() throws Exception {
+
+ logger.debug("testIntegration100EntitiesOn(): starting...");
+
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ // create five applications each with collection of five entities
+
+ for ( int i = 0; i < 5; i++ ) {
+
+ ApplicationInfo appMade = setup.getMgmtSvc().createApplication( organization.getUuid(), "superapp" + i );
+ EntityManager appEm = setup.getEmf().getEntityManager( appMade.getId() );
+
+ String collName = "superappCol" + i;
+ appEm.createApplicationCollection(collName);
+
+ Map<String, Object> entityLevelProperties = null;
+ Entity[] entNotCopied;
+ entNotCopied = new Entity[5];
+
+ for ( int index = 0; index < 5; index++ ) {
+ entityLevelProperties = new LinkedHashMap<String, Object>();
+ entityLevelProperties.put( "username", "bobso" + index );
+ entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
+ entNotCopied[index] = appEm.create( collName, entityLevelProperties );
+ }
+ }
+
+ // export the organization containing those apps and collections
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ int maxRetries = 100;
+ int retries = 0;
+ while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) && retries++ < maxRetries ) {
+ Thread.sleep(100);
+ }
+
+ String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+ String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
+ Properties overrides = new Properties();
+ overrides.setProperty( "s3" + ".identity", accessId );
+ overrides.setProperty( "s3" + ".credential", secretKey );
+
+ // test that we can find the file that were exported to S3
+
+ BlobStore blobStore = null;
+ try {
+
+ final Iterable<? extends Module> MODULES = ImmutableSet.of(
+ new JavaUrlHttpCommandExecutorServiceModule(),
+ new Log4JLoggingModule(),
+ new NettyPayloadModule());
+
+ BlobStoreContext context = ContextBuilder.newBuilder("s3")
+ .credentials(accessId, secretKey)
+ .modules(MODULES)
+ .overrides(overrides)
+ .buildView(BlobStoreContext.class);
+
+ String expectedFileName = ((ExportServiceImpl) exportService)
+ .prepareOutputFileName(organization.getName(), "applications");
+
+ blobStore = context.getBlobStore();
+ if (!blobStore.blobExists(bucketName, expectedFileName)) {
+ blobStore.deleteContainer(bucketName);
+ Assert.fail("Blob does not exist: " + expectedFileName);
+ }
+ Blob bo = blobStore.getBlob(bucketName, expectedFileName);
+
+ Long numOfFiles = blobStore.countBlobs(bucketName);
+ Long numWeWant = 1L;
+ blobStore.deleteContainer(bucketName);
+ assertEquals(numOfFiles, numWeWant);
+ assertNotNull(bo);
+
+ } finally {
+ blobStore.deleteContainer(bucketName);
+ }
+ }
+
+ @Ignore("Why is this ignored?")
+ @Test
+ public void testIntegration100EntitiesForAllApps() throws Exception {
+
+ S3Export s3Export = new S3ExportImpl();
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ OrganizationInfo orgMade = null;
+ ApplicationInfo appMade = null;
+ for ( int i = 0; i < 5; i++ ) {
+ orgMade = setup.getMgmtSvc().createOrganization( "minorboss" + i, adminUser, true );
+ for ( int j = 0; j < 5; j++ ) {
+ appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + j );
+
+ EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
+ customMaker.createApplicationCollection( "superappCol" + j );
+ //intialize user object to be posted
+ Map<String, Object> entityLevelProperties = null;
+ Entity[] entNotCopied;
+ entNotCopied = new Entity[1];
+ //creates entities
+ for ( int index = 0; index < 1; index++ ) {
+ entityLevelProperties = new LinkedHashMap<String, Object>();
+ entityLevelProperties.put( "derp", "bacon" );
+ entNotCopied[index] = customMaker.create( "superappCol" + j, entityLevelProperties );
+ }
+ }
+ }
+
+ payload.put( "organizationId", orgMade.getUuid() );
+
+ UUID exportUUID = exportService.schedule( payload );
+ assertNotNull( exportUUID );
+
+ Thread.sleep( 3000 );
+
+ String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+ String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
+
+ Properties overrides = new Properties();
+ overrides.setProperty( "s3" + ".identity", accessId );
+ overrides.setProperty( "s3" + ".credential", secretKey );
+
+ BlobStore blobStore = null;
+
+ try {
+ final Iterable<? extends Module> MODULES = ImmutableSet.of(
+ new JavaUrlHttpCommandExecutorServiceModule(),
+ new Log4JLoggingModule(),
+ new NettyPayloadModule() );
+
+ BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
+ .credentials(accessId, secretKey )
+ .modules(MODULES )
+ .overrides(overrides )
+ .buildView(BlobStoreContext.class );
+
+ blobStore = context.getBlobStore();
+
+ //Grab Number of files
+ Long numOfFiles = blobStore.countBlobs( bucketName );
+
+ String expectedFileName = ((ExportServiceImpl)exportService)
+ .prepareOutputFileName(organization.getName(), "applications");
+
+ //delete container containing said files
+ Blob bo = blobStore.getBlob(bucketName, expectedFileName);
+ Long numWeWant = 5L;
+ blobStore.deleteContainer( bucketName );
+
+ //asserts that the correct number of files was transferred over
+ assertEquals( numWeWant, numOfFiles );
+
+ }
+ finally {
+ blobStore.deleteContainer( bucketName );
+ }
+ }
+
+
+ @Ignore("Why is this ignored")
+ @Test
+ public void testIntegration100EntitiesOnOneOrg() throws Exception {
+
+ S3Export s3Export = new S3ExportImpl();
+ ExportService exportService = setup.getExportService();
+
+ String appName = newOrgAppAdminRule.getApplicationInfo().getName();
+ HashMap<String, Object> payload = payloadBuilder(appName);
+
+ payload.put( "organizationId", organization.getUuid() );
+ payload.put( "applicationId", applicationId );
+
+ OrganizationInfo orgMade = null;
+ ApplicationInfo appMade = null;
+ for ( int i = 0; i < 100; i++ ) {
+ orgMade = setup.getMgmtSvc().createOrganization( "largerboss" + i, adminUser, true );
+ appMade = setup.getMgmtSvc().createApplication( orgMade.getUuid(), "superapp" + i );
+
+ EntityManager customMaker = setup.getEmf().getEntityManager( appMade.getId() );
+ customMaker.createApplicationCollection( "superappCol" + i );
+ //intialize user object to be posted
+ Map<String, Object> entityLevelProperties = null;
+ Entity[] entNotCopied;
+ entNotCopied = new Entity[20];
+ //creates entities
+ for ( int index = 0; index < 20; index++ ) {
+ entityLevelProperties = new LinkedHashMap<String, Object>();
+ entityLevelProperties.put( "username", "bobso" + index );
+ entityLevelProperties.put( "email", "derp" + index + "@anuff.com" );
+ entNotCopied[index] = customMaker.create( "superappCol", entityLevelProperties );
+ }
+ }
+
+ EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+ //intialize user object to be posted
+ Map<String, Object> userProperties = null;
+ Entity[] entity;
+ entity = new Entity[100];
+
+ //creates entities
+ for ( int i = 0; i < 100; i++ ) {
+ userProperties = new LinkedHashMap<String, Object>();
+ userProperties.put( "username", "bido" + i );
+ userProperties.put( "email", "bido" + i + "@anuff.com" );
+
+ entity[i] = em.create( "user", userProperties );
+ }
+
+ UUID exportUUID = exportService.schedule( payload );
+
+ while ( !exportService.getState( exportUUID ).equals( "FINISHED" ) ) {}
+
+ String accessId = System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR );
+ String secretKey = System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR );
+
+ Properties overrides = new Properties();
+ overrides.setProperty( "s3" + ".identity", accessId );
+ overrides.setProperty( "s3" + ".credential", secretKey );
+
+ Blob bo = null;
+ BlobStore blobStore = null;
+
+ try {
+ final Iterable<? extends Module> MODULES = ImmutableSet.of(
+ new JavaUrlHttpCommandExecutorServiceModule(),
+ new Log4JLoggingModule(),
+ new NettyPayloadModule() );
+
+ BlobStoreContext context = ContextBuilder.newBuilder( "s3" )
+ .credentials(accessId, secretKey )
+ .modules(MODULES )
+ .overrides(overrides )
+ .buildView(BlobStoreContext.class );
+
+ String expectedFileName = ((ExportServiceImpl)exportService)
+ .prepareOutputFileName(organization.getName(), "applications");
+
+ blobStore = context.getBlobStore();
+ if ( !blobStore.blobExists( bucketName, expectedFileName ) ) {
+ assert ( false );
+ }
+ Long numOfFiles = blobStore.countBlobs( bucketName );
+ Long numWeWant = Long.valueOf( 1 );
+ assertEquals( numOfFiles, numWeWant );
+
+ bo = blobStore.getBlob( bucketName, expectedFileName );
+ }
+ catch ( Exception e ) {
+ assert ( false );
+ }
+
+ assertNotNull( bo );
+ blobStore.deleteContainer( bucketName );
+ }
+
+ public JobData jobDataCreator(HashMap<String, Object> payload,UUID exportUUID, S3Export s3Export) {
+ JobData jobData = new JobData();
+
+ jobData.setProperty( "jobName", "exportJob" );
+ jobData.setProperty( "exportInfo", payload );
+ jobData.setProperty( "exportId", exportUUID );
+ jobData.setProperty( "s3Export", s3Export );
+
+ return jobData;
+ }
+
+ /*Creates fake payload for testing purposes.*/
+ public HashMap<String, Object> payloadBuilder( String orgOrAppName ) {
+ HashMap<String, Object> payload = new HashMap<String, Object>();
+ Map<String, Object> properties = new HashMap<String, Object>();
+ Map<String, Object> storage_info = new HashMap<String, Object>();
+ storage_info.put( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR,
+ System.getProperty( SDKGlobalConfiguration.SECRET_KEY_ENV_VAR ) );
+ storage_info.put( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR,
+ System.getProperty( SDKGlobalConfiguration.ACCESS_KEY_ENV_VAR ) );
+ storage_info.put( "bucket_location", bucketName );
+
+ properties.put( "storage_provider", "s3" );
+ properties.put( "storage_info", storage_info );
+
+ payload.put( "path", orgOrAppName );
+ payload.put( "properties", properties );
+ return payload;
+ }
+ }