You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/02/16 20:06:04 UTC
[13/50] incubator-usergrid git commit: Merge branch
'two-dot-o-import' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
two-dot-o-import
Merge branch 'two-dot-o-import' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into two-dot-o-import
Conflicts:
stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a4327e81
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a4327e81
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a4327e81
Branch: refs/heads/USERGRID-273
Commit: a4327e8152162288e7881ecdb7a243b6ea7ae9fd
Parents: 0234e2b 25b3645
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Feb 11 13:14:28 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Feb 11 13:14:28 2015 -0500
----------------------------------------------------------------------
.../usergrid/persistence/index/query/Query.java | 47 +++--
.../org/apache/usergrid/rest/ApiResponse.java | 16 ++
.../exceptions/NullArgumentExceptionMapper.java | 42 +++++
.../imports/FileErrorsResource.java | 143 ++++++++++++++++
.../imports/FileIncludesResource.java | 159 +++++++++++++++++
.../applications/imports/ImportResource.java | 98 -----------
.../applications/imports/ImportsResource.java | 126 ++++++++------
.../rest/management/ImportResourceIT.java | 104 +++++------
.../management/importer/FileImportTracker.java | 3 +
.../management/importer/ImportService.java | 52 +++++-
.../management/importer/ImportServiceImpl.java | 171 ++++++++++++++++---
11 files changed, 717 insertions(+), 244 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a4327e81/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index 6f08ffb,f35e6f2..d549dba
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@@ -78,16 -80,7 +80,6 @@@ public class ImportServiceImpl implemen
}
- public SchedulerService getSch() {
- return sch;
- }
-
-
- public void setSch(final SchedulerService sch) {
- this.sch = sch;
- }
-
--
/**
* This schedules the main import Job.
*
@@@ -181,6 -195,118 +194,118 @@@
return entity;
}
+ @Override
+ public Results getFileImports(final UUID applicationId, final UUID importId, @Nullable final String ql, @Nullable final String cursor ) {
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+ Preconditions.checkNotNull( importId, "importId must be specified" );
+
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
+ final Import importEntity = getImport( applicationId, importId );
+
+ Query query = Query.fromQLNullSafe( ql );
+ query.setCursor( cursor );
+
+ //set our entity type
+ query.setEntityType( Schema.getDefaultSchema().getEntityType( Import.class ) );
+
+ return rootEm.searchCollection( importEntity, IMPORT_FILE_INCLUDES_CONNECTION, query );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to get import entity", e );
+ }
+
+ }
+
+
+ @Override
+ public FileImport getFileImport(final UUID applicationId, final UUID importId, final UUID fileImportId ) {
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
+ final Import importEntity = getImport( applicationId, importId );
+
+ if ( importEntity == null ) {
+ throw new EntityNotFoundException( "Import not found with id " + importId );
+ }
+
+
+ final FileImport fileImport = rootEm.get( importId, FileImport.class );
+
+
+ // check if it's on the path
+ if ( !rootEm.isConnectionMember( importEntity, APP_IMPORT_CONNECTION, fileImport ) ) {
+ return null;
+ }
+
+ return fileImport;
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to load file import", e );
+ }
+ }
+
+
+ @Override
+ public Results getFailedImportEntities(final UUID applicationId, final UUID importId, final UUID fileImportId, @Nullable final String ql, @Nullable final String cursor ) {
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+ Preconditions.checkNotNull( importId, "importId must be specified" );
+ Preconditions.checkNotNull( fileImportId, "fileImportId must be specified" );
+
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
- final FileImport importEntity = getFileImport( applicationId, importId, fileImportId );
++ final FileImport importEntity = getFileImport(applicationId, importId, fileImportId);
+
+ Query query = Query.fromQLNullSafe( ql );
+ query.setCursor( cursor );
+
+ //set our entity type
+ query.setEntityType( Schema.getDefaultSchema().getEntityType( FailedImportEntity.class ) );
+
+ return rootEm.searchCollection( importEntity, FileImportTracker.ERRORS_CONNECTION_NAME, query );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to get import entity", e );
+ }
+ }
+
+
+ @Override
+ public FailedImportEntity getFailedImportEntity(final UUID applicationId, final UUID importId, final UUID fileImportId,
+ final UUID failedImportId ) {
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
+ final FileImport importEntity = getFileImport( applicationId, importId, fileImportId );
+
+ if ( importEntity == null ) {
+ throw new EntityNotFoundException( "Import not found with id " + importId );
+ }
+
+
+ final FailedImportEntity fileImport = rootEm.get( importId, FailedImportEntity.class );
+
+
+ // check if it's on the path
+ if ( !rootEm.isConnectionMember( importEntity, FileImportTracker.ERRORS_CONNECTION_NAME, fileImport ) ) {
+ return null;
+ }
+
+ return fileImport;
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to load file import", e );
+ }
+ }
+
/**
* This schedules the sub FileImport Job
@@@ -343,7 -470,7 +469,7 @@@
UUID importId = (UUID) jobExecution.getJobData().getProperty(IMPORT_ID);
EntityManager importManager = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
-- return importManager.get( importId, Import.class );
++ return importManager.get(importId, Import.class);
}
@@@ -651,54 -728,42 +782,54 @@@
// get parent import job of this file import job
- Results importJobResults =
- emManagementApp.getConnectingEntities( fileImport, IMPORT_FILE_INCLUDES_CONNECTION, null, Level.ALL_PROPERTIES );
- List<Entity> importEntities = importJobResults.getEntities();
- UUID importId = importEntities.get( 0 ).getUuid();
- Import importEntity = emManagementApp.get( importId, Import.class );
+ String randTag = RandomStringUtils.randomAlphanumeric(4); // for logging
- String randTag = RandomStringUtils.randomAlphanumeric(4);
- logger.debug("{} Got importEntity {}", randTag, importEntity.getUuid() );
+ int failCount = 0;
+ int successCount = 0;
+ Import importEntity = null;
+ try {
- EntityManager emMgmtApp = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- Query query = Query.fromQL("select *");
- query.setEntityType("file_import");
- query.setConnectionType( IMPORT_FILE_INCLUDES_CONNECTION );
- query.setLimit(MAX_FILE_IMPORTS);
- Results entities = emMgmtApp.searchConnectedEntities(importEntity, query);
+ Results importJobResults =
- emManagementApp.getConnectingEntities(fileImport, IMPORT_FILE_INCLUDES, null, Level.ALL_PROPERTIES);
++ emManagementApp.getConnectingEntities( fileImport, IMPORT_FILE_INCLUDES_CONNECTION, null, Level.ALL_PROPERTIES );
+ List<Entity> importEntities = importJobResults.getEntities();
+ UUID importId = importEntities.get(0).getUuid();
+ importEntity = emManagementApp.get(importId, Import.class);
- PagingResultsIterator itr = new PagingResultsIterator( entities );
+ logger.debug("{} Got importEntity {}", randTag, importEntity.getUuid());
- EntityManager emMgmtApp = emf.getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
- logger.debug("{} Check {} jobs to see if we are done for file {}",
- new Object[] { randTag, entities.size(), fileImport.getFileName() } );
++ EntityManager emMgmtApp = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
+ Query query = Query.fromQL("select *");
+ query.setEntityType("file_import");
- query.setConnectionType(IMPORT_FILE_INCLUDES);
++ query.setConnectionType( IMPORT_FILE_INCLUDES_CONNECTION );
+ query.setLimit(MAX_FILE_IMPORTS);
+ Results entities = emMgmtApp.searchConnectedEntities(importEntity, query);
+
+ PagingResultsIterator itr = new PagingResultsIterator(entities);
+
+ logger.debug("{} Check {} jobs to see if we are done for file {}",
+ new Object[]{randTag, entities.size(), fileImport.getFileName()});
+
+ while (itr.hasNext()) {
+ FileImport fi = (FileImport) itr.next();
+ switch (fi.getState()) {
+ case FAILED: // failed, but we may not be complete so continue checking
+ failCount++;
+ break;
+ case FINISHED: // finished, we can continue checking
+ successCount++;
+ continue;
+ default: // not something we recognize as complete, short circuit
+ logger.debug("{} not done yet, bail out...", randTag);
+ return;
+ }
+ }
- int failCount = 0;
- int successCount = 0;
- while ( itr.hasNext() ) {
- FileImport fi = ( FileImport ) itr.next();
- switch ( fi.getState() ) {
- case FAILED: // failed, but we may not be complete so continue checking
- failCount++;
- break;
- case FINISHED: // finished, we can continue checking
- successCount++;
- continue;
- default: // not something we recognize as complete, short circuit
- logger.debug("{} not done yet, bail out...", randTag);
- return;
+ } catch ( Exception e ) {
+ failCount++;
+ if ( importEntity != null ) {
+ importEntity.setErrorMessage("Error determining status of file import jobs");
}
+ logger.debug("Error determining status of file import jobs", e);
}
logger.debug("{} successCount = {} failCount = {}", new Object[] { randTag, successCount, failCount } );