You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/03/10 21:26:17 UTC
[24/50] [abbrv] git commit: Added different states. New States that
are in use are currently: CREATED, FAILED, SCHEDULED, STARTED,
FINISHED. Support for canceled and expired at a later date.
Added different states. New States that are in use are currently: CREATED,FAILED,SCHEDULED,STARTED,FINISHED. Support for canceled and expired at a later date.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/41caff77
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/41caff77
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/41caff77
Branch: refs/pull/70/merge
Commit: 41caff773cb5165ff21252903a66710970564a5c
Parents: 96a1a72
Author: grey <gr...@apigee.com>
Authored: Tue Feb 25 14:01:40 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Feb 25 14:01:40 2014 -0800
----------------------------------------------------------------------
.../usergrid/persistence/entities/Export.java | 5 +-
.../rest/management/ManagementResourceIT.java | 2 +-
.../usergrid/management/export/ExportJob.java | 33 +++++++--
.../management/export/ExportServiceImpl.java | 77 +++++++-------------
.../cassandra/ManagementServiceIT.java | 32 ++++----
5 files changed, 70 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
index 7905550..1719412 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Export.java
@@ -14,10 +14,9 @@ import org.apache.usergrid.persistence.annotations.EntityProperty;
*/
@XmlRootElement
public class Export extends TypedEntity {
-
+ //canceled , and expired states aren't used in current iteration.
public static enum State {
- //CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
- PENDING, STARTED, FAILED, COMPLETED
+ CREATED, FAILED, SCHEDULED, STARTED, FINISHED, CANCELED, EXPIRED
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
index a9b06fd..a36dfd3 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/ManagementResourceIT.java
@@ -645,7 +645,7 @@ public class ManagementResourceIT extends AbstractRestIT {
assertEquals( Status.OK, responseStatus );
- assertEquals( "PENDING", node.asText() );//TODO: do tests for other states in service tier
+ assertEquals( "SCHEDULED", node.asText() );//TODO: do tests for other states in service tier
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
index ed2fef5..83cda0a 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportJob.java
@@ -1,6 +1,8 @@
package org.apache.usergrid.management.export;
+import java.util.UUID;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -9,6 +11,9 @@ import org.springframework.stereotype.Component;
import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.batch.job.OnlyOnceJob;
import org.apache.usergrid.management.ExportInfo;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.persistence.entities.JobData;
@@ -18,36 +23,52 @@ import org.apache.usergrid.persistence.entities.JobData;
*/
@Component("exportJob")
public class ExportJob extends OnlyOnceJob {
+ public static final String EXPORT_ID = "exportId";
private static final Logger logger = LoggerFactory.getLogger( ExportJob.class );
@Autowired
ExportService exportService;
+ @Autowired
+ private EntityManagerFactory emf;
+
public ExportJob() {
logger.info( "ExportJob created " + this );
}
-
+//need to add in checking the export state and what happens if it fails. This should then update the export entity with failed state.
@Override
public void doJob( JobExecution jobExecution ) throws Exception {
logger.info( "execute ExportJob {}", jobExecution );
+ UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+ //as long as I have the entity UUID I should be able to find it from anywhere right?
+
+
ExportInfo config = null;
JobData jobData = jobExecution.getJobData();
- config = ( ExportInfo ) jobData.getProperty( "exportInfo" );
if ( jobData == null ) {
logger.error( "jobData cannot be null" );
return;
}
- else if ( config == null ) {
+ config = ( ExportInfo ) jobData.getProperty( "exportInfo" );
+ if ( config == null ) {
logger.error( "Export information cannot be null" );
return;
}
- else {
+ EntityManager em = emf.getEntityManager( config.getApplicationId() );
+ Export export = em.get( exportId, Export.class );
+
jobExecution.heartbeat();
- exportService.doExport( config, jobExecution );
- }
+ try {
+ exportService.doExport( config, jobExecution );
+ }catch (Exception e) {
+ logger.error( "Export Service failed to complete job" );
+ export.setState( Export.State.FAILED );
+ em.update( export );
+ }
+
logger.info( "executed ExportJob process completed" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
index 155639c..37646a5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/export/ExportServiceImpl.java
@@ -75,39 +75,18 @@ public class ExportServiceImpl implements ExportService {
@Override
public UUID schedule( final ExportInfo config ) throws Exception {
- EntityManager em = emf.getEntityManager( config.getApplicationId() );
-
- Export export = new Export();
- export.setState( Export.State.PENDING );
-
- //validate that org exists,then app, then collection.
-
- String pathToBeParsed = config.getPath();
- //split the path so that you can verify that the organization and the app exist.
- String[] pathItems = pathToBeParsed.split( "/" );
-
-
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- }
-
- try {
- managementService.getApplicationInfo( pathItems[1] );
- }
- catch ( Exception e ) {
- logger.error( "Application doesn't exist" );
+ if(config == null){
+ logger.error( "export information cannot be null" );
+ return null;
}
+ EntityManager em = emf.getEntityManager( config.getApplicationId() );
- //TODO: parse path and make sure all the things you need actually exist. then throw
- // good error messages when not found.
+ Export export = new Export();
//write to the em
export = em.create( export );
- export.setState( Export.State.PENDING );
+ export.setState( Export.State.CREATED );
em.update( export );
JobData jobData = new JobData();
@@ -117,6 +96,8 @@ public class ExportServiceImpl implements ExportService {
long soonestPossible = System.currentTimeMillis() + 250; //sch grace period
sch.createJob( EXPORT_JOB_NAME, soonestPossible, jobData );
+ export.setState( Export.State.SCHEDULED );
+ em.update( export );
return export.getUuid();
}
@@ -147,34 +128,25 @@ public class ExportServiceImpl implements ExportService {
EntityManager em = emf.getEntityManager( config.getApplicationId() );
Export export = em.get( exportId, Export.class );
- String pathToBeParsed = config.getPath();
- String[] pathItems = pathToBeParsed.split( "/" );
- try {
- managementService.getOrganizationByName( pathItems[0] );
- }
- catch ( Exception e ) {
- logger.error( "Organization doesn't exist" );
- return;
- }
-
- //update state and re-write the entity
export.setState( Export.State.STARTED );
-
em.update( export );
Map<UUID, String> organizationGet = getOrgs( config );
for ( Map.Entry<UUID, String> organization : organizationGet.entrySet() ) {
- //needs to pass app name, and possibly collection to export
- exportApplicationsForOrg( organization, config, jobExecution );
+ try {
+ exportApplicationsForOrg( organization, config, jobExecution );
+ }catch(Exception e){
+ export.setState( Export.State.FAILED );
+ return;
+ }
}
- export.setState( Export.State.COMPLETED );
+ export.setState( Export.State.FINISHED );
em.update( export );
}
private Map<UUID, String> getOrgs( ExportInfo exportInfo ) throws Exception {
// Loop through the organizations
- // TODO:this will come from the orgs in schedule when you do the validations. delete orgId
UUID orgId = null;
Map<UUID, String> organizationNames = null;
@@ -183,7 +155,7 @@ public class ExportServiceImpl implements ExportService {
organizationNames = managementService.getOrganizations();
}
- else {
+ else {//this case isn't used yet, but might be in the future.
OrganizationInfo info = managementService.getOrganizationByUuid( orgId );
if ( info == null ) {
@@ -194,7 +166,6 @@ public class ExportServiceImpl implements ExportService {
organizationNames.put( orgId, info.getName() );
}
-
return organizationNames;
}
@@ -236,6 +207,10 @@ public class ExportServiceImpl implements ExportService {
private void exportApplicationsForOrg( Map.Entry<UUID, String> organization, final ExportInfo config,
final JobExecution jobExecution ) throws Exception {
+ UUID exportId = ( UUID ) jobExecution.getJobData().getProperty( EXPORT_ID );
+ EntityManager exportManager = emf.getEntityManager( config.getApplicationId() );
+ Export export = exportManager.get( exportId, Export.class );
+
ByteArrayOutputStream baos = new ByteArrayOutputStream();
logger.info( "" + organization );
@@ -291,7 +266,12 @@ public class ExportServiceImpl implements ExportService {
baos.close();
InputStream is = new ByteArrayInputStream( baos.toByteArray() );
- s3Export.copyToS3( is, config, appFileName );
+ try {
+ s3Export.copyToS3( is, config, appFileName );
+ }catch(Exception e){
+ export.setState( Export.State.FAILED );
+ return;
+ }
}
}
@@ -319,14 +299,12 @@ public class ExportServiceImpl implements ExportService {
throws Exception {
Set<String> collections = em.getCollections( entity );
- //jg.writeStartObject();
// Only create entry for Entities that have collections
if ( ( collections == null ) || collections.isEmpty() ) {
return;
}
-
for ( String collectionName : collections ) {
jg.writeFieldName( collectionName );
@@ -354,8 +332,6 @@ public class ExportServiceImpl implements ExportService {
// Write dictionaries
saveDictionaries( entity, em, jg );
- // End the object if it was Started
- //jg.writeEndObject();
}
@@ -448,7 +424,6 @@ public class ExportServiceImpl implements ExportService {
str.append( ".json" );
String outputFileName = str.toString();
- //TODO:this is , i feel, bad practice so make sure to come back here and fix it up.
return outputFileName;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41caff77/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
index 34affe6..0b1f7be 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/cassandra/ManagementServiceIT.java
@@ -18,13 +18,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.management.ExportInfo;
-import org.apache.usergrid.management.export.ExportJob;
-import org.apache.usergrid.management.export.ExportService;
-import org.apache.usergrid.management.export.S3Export;
-import org.apache.usergrid.management.export.S3ExportImpl;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.ServiceITSetup;
import org.apache.usergrid.ServiceITSetupImpl;
@@ -33,11 +26,19 @@ import org.apache.usergrid.batch.JobExecution;
import org.apache.usergrid.cassandra.CassandraResource;
import org.apache.usergrid.cassandra.ClearShiroSubject;
import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.count.SimpleBatcher;
+import org.apache.usergrid.management.ExportInfo;
import org.apache.usergrid.management.OrganizationInfo;
import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.management.export.ExportJob;
+import org.apache.usergrid.management.export.ExportService;
+import org.apache.usergrid.management.export.S3Export;
+import org.apache.usergrid.management.export.S3ExportImpl;
import org.apache.usergrid.persistence.CredentialsInfo;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.entities.Export;
import org.apache.usergrid.persistence.entities.JobData;
import org.apache.usergrid.persistence.entities.User;
import org.apache.usergrid.security.AuthPrincipalType;
@@ -48,16 +49,14 @@ import org.apache.usergrid.security.tokens.exceptions.InvalidTokenException;
import org.apache.usergrid.utils.JsonUtils;
import org.apache.usergrid.utils.UUIDUtils;
-import org.apache.usergrid.count.SimpleBatcher;
-
import static org.apache.commons.codec.binary.Base64.encodeBase64URLSafeString;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_CREDENTIALS;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.MANAGEMENT_APPLICATION_ID;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -862,7 +861,6 @@ public class ManagementServiceIT {
f.delete();
}
-
@Test
public void testFileExportOneOrg() throws Exception {
@@ -1013,7 +1011,6 @@ public class ManagementServiceIT {
assert ( true );
}
-
@Test
public void testExportDoExport() throws Exception {
@@ -1041,20 +1038,19 @@ public class ManagementServiceIT {
Export exportEntity = ( Export ) em.get( entityExportUUID );
assertNotNull( exportEntity );
String derp = exportEntity.getState().name();
- assertEquals( "PENDING", exportEntity.getState().name() );
+ assertEquals( "SCHEDULED", exportEntity.getState().name() );
try {
eS.doExport( exportInfo, jobExecution );
}
catch ( Exception e ) {
- throw e;
- //assert(false);
+ //throw e;
+ assert(false);
}
exportEntity = ( Export ) em.get( entityExportUUID );
assertNotNull( exportEntity );
- assertEquals( "COMPLETED", exportEntity.getState().name() );
+ assertEquals( "FINISHED", exportEntity.getState().name() );
}
-
//tests that with empty job data, the export still runs.
@Test
public void testExportEmptyJobData() throws Exception {