You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/02/11 18:23:56 UTC
[3/3] incubator-usergrid git commit: Updated resources and services
to all standard service GET/query semantics
Updated resources and services to all standard service GET/query semantics
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/db8184c8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/db8184c8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/db8184c8
Branch: refs/heads/two-dot-o-import
Commit: db8184c845231a8f807249c1cd3e03b9f5f169df
Parents: c632756
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Feb 11 10:23:51 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Feb 11 10:23:51 2015 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/query/Query.java | 47 ++++---
.../imports/FileErrorsResource.java | 4 +-
.../imports/FileIncludesResource.java | 26 +---
.../applications/imports/ImportsResource.java | 4 +-
.../management/importer/FileImportTracker.java | 3 +
.../management/importer/ImportService.java | 33 +++--
.../management/importer/ImportServiceImpl.java | 129 ++++++++++++++++---
7 files changed, 172 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db8184c8/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 414061d..da68772 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -116,12 +116,12 @@ public class Query {
public Query( Query q ) {
if ( q != null ) {
type = q.type;
- sortPredicates = q.sortPredicates != null
+ sortPredicates = q.sortPredicates != null
? new ArrayList<SortPredicate>( q.sortPredicates ) : null;
startResult = q.startResult;
cursor = q.cursor;
limit = q.limit;
- selectAssignments = q.selectAssignments != null
+ selectAssignments = q.selectAssignments != null
? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
mergeSelectResults = q.mergeSelectResults;
//level = q.level;
@@ -134,9 +134,9 @@ public class Query {
resolution = q.resolution;
pad = q.pad;
rootOperand = q.rootOperand;
- identifiers = q.identifiers != null
+ identifiers = q.identifiers != null
? new ArrayList<Identifier>( q.identifiers ) : null;
- counterFilters = q.counterFilters != null
+ counterFilters = q.counterFilters != null
? new ArrayList<CounterFilterPredicate>( q.counterFilters ) : null;
collection = q.collection;
}
@@ -151,8 +151,8 @@ public class Query {
//we have a root operand. Translate our AST into an ES search
if ( getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
+ // In the case of geo only queries, this will return null into the query builder.
+ // Once we start using tiles, we won't need this check any longer, since a geo query
// will return a tile query + post filter
QueryVisitor v = new EsQueryVistor();
@@ -168,11 +168,11 @@ public class Query {
}
- // Add our filter for context to our query for fast execution.
+ // Add our filter for context to our query for fast execution.
// Fast because it utilizes bitsets internally. See this post for more detail.
// http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
- // TODO evaluate performance when it's an all query.
+ // TODO evaluate performance when it's an all query.
// Do we need to put the context term first for performance?
if ( queryBuilder != null ) {
queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
@@ -200,12 +200,27 @@ public class Query {
throw new RuntimeException( "Error building ElasticSearch query", ex );
}
filterBuilder = v.getFilterBuilder();
- }
+ }
- return filterBuilder;
+ return filterBuilder;
}
+ /**
+ * Create a query instance from the QL. If the string is null, return an empty query
+ * @param ql
+ * @return
+ */
+ public static Query fromQLNullSafe(final String ql){
+ final Query query = fromQL(ql);
+
+ if(query != null){
+ return query;
+ }
+
+ return new Query();
+ }
+
public static Query fromQL( String ql ) throws QueryParseException {
if ( StringUtils.isEmpty(ql) ) {
return null;
@@ -215,8 +230,8 @@ public class Query {
ql = ql.trim();
String qlt = ql.toLowerCase();
- if ( !qlt.startsWith( "select" )
- && !qlt.startsWith( "insert" )
+ if ( !qlt.startsWith( "select" )
+ && !qlt.startsWith( "insert" )
&& !qlt.startsWith( "update" ) && !qlt.startsWith( "delete" ) ) {
if ( qlt.startsWith( "order by" ) ) {
@@ -278,7 +293,7 @@ public class Query {
}
- public static Query fromQueryParams( Map<String, List<String>> params )
+ public static Query fromQueryParams( Map<String, List<String>> params )
throws QueryParseException {
Query q = null;
CounterResolution resolution = null;
@@ -636,7 +651,7 @@ public class Query {
for ( SortPredicate s : sortPredicates ) {
if ( s.getPropertyName().equals( sort.getPropertyName() ) ) {
- throw new QueryParseException( String.format(
+ throw new QueryParseException( String.format(
"Attempted to set sort order for %s more than once", s.getPropertyName() ) );
}
}
@@ -687,7 +702,7 @@ public class Query {
for ( SortPredicate s : sortPredicates ) {
if ( s.getPropertyName().equals( propertyName ) ) {
logger.error(
- "Attempted to set sort order for " + s.getPropertyName()
+ "Attempted to set sort order for " + s.getPropertyName()
+ " more than once, discarding..." );
return this;
}
@@ -1110,7 +1125,7 @@ public class Query {
private final Query.SortDirection direction;
- public SortPredicate(@JsonProperty("propertyName") String propertyName,
+ public SortPredicate(@JsonProperty("propertyName") String propertyName,
@JsonProperty("direction") Query.SortDirection direction ) {
if ( propertyName == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db8184c8/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java
index 552b3b5..a72a8f0 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileErrorsResource.java
@@ -82,12 +82,12 @@ public class FileErrorsResource extends AbstractContextResource {
@GET
- public JSONWithPadding getFileIncludes( @Context UriInfo ui, @QueryParam( "cursor" ) String cursor )
+ public JSONWithPadding getFileIncludes( @Context UriInfo ui, @QueryParam( "ql" ) String query, @QueryParam( "cursor" ) String cursor )
throws Exception {
final Results importResults = importService.getFailedImportEntities( application.getId(), importId,
- importFileId, cursor );
+ importFileId, query, cursor );
if(importResults == null){
throw new EntityNotFoundException( "could not load import results" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db8184c8/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java
index 2d14e93..c0cfe79 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/FileIncludesResource.java
@@ -89,11 +89,11 @@ public class FileIncludesResource extends AbstractContextResource {
@GET
- public JSONWithPadding getFileIncludes( @Context UriInfo ui, @QueryParam( "cursor" ) String cursor )
+ public JSONWithPadding getFileIncludes( @Context UriInfo ui, @QueryParam( "ql" ) String query, @QueryParam( "cursor" ) String cursor )
throws Exception {
- final Results importResults = importService.getFileImports( application.getId(), importId, cursor );
+ final Results importResults = importService.getFileImports( application.getId(), importId, query, cursor );
if(importResults == null){
throw new EntityNotFoundException( "could not load import results" );
@@ -145,27 +145,11 @@ public class FileIncludesResource extends AbstractContextResource {
@GET
@Path( RootResource.ENTITY_ID_PATH + "/errors" )
- public JSONWithPadding getIncludes( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId )
+ public FileErrorsResource getIncludes( @Context UriInfo ui, @PathParam( "entityId" ) PathSegment entityId )
throws Exception {
- final UUID importId = UUID.fromString( entityId.getPath() );
- final Import importEntity = importService.getImport( application.getId(), importId);
-
- if(importEntity == null){
- throw new EntityNotFoundException( "could not find import with uuid " + importId );
- }
-
- ApiResponse response = createApiResponse();
-
-
- response.setAction( "get" );
- response.setApplication( emf.getEntityManager( application.getId() ).getApplication() );
- response.setParams( ui.getQueryParameters() );
-
-
- response.setEntities( Collections.<Entity>singletonList( importEntity ) );
-
- return new JSONWithPadding( response );
+ final UUID fileImportId = UUID.fromString( entityId.getPath() );
+ return getSubResource( FileErrorsResource.class ).init( application, importId,fileImportId );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db8184c8/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
index 8d23bcd..86bf278 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/management/organizations/applications/imports/ImportsResource.java
@@ -144,10 +144,10 @@ public class ImportsResource extends AbstractContextResource {
@GET
- public JSONWithPadding getImports( @Context UriInfo ui, @QueryParam( "cursor" ) String cursor ) throws Exception {
+ public JSONWithPadding getImports( @Context UriInfo ui, @QueryParam( "ql" ) String query, @QueryParam( "cursor" ) String cursor ) throws Exception {
- final Results importResults = importService.getImports( application.getId(), cursor );
+ final Results importResults = importService.getImports( application.getId(), query, cursor );
if ( importResults == null ) {
throw new EntityNotFoundException( "could not load import results" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db8184c8/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
index be2f72b..726bceb 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/FileImportTracker.java
@@ -43,6 +43,9 @@ public class FileImportTracker {
private static final String ERROR_MESSAGE =
"Failed to import some data. See the import counters and errors.";
+ /**
+ * Connection name to log individual errors
+ */
public static final String ERRORS_CONNECTION_NAME = "errors";
private final AtomicLong entitiesWritten = new AtomicLong( 0 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db8184c8/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportService.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportService.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportService.java
index d899dc9..d63c076 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportService.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportService.java
@@ -41,10 +41,11 @@ public interface ImportService {
/**
* Get the imports results for the application
* @param applicationId
- * @param cursor
+ * @param ql The query executed (nullable)
+ * @param cursor The cursor passed (nullable)
* @return
*/
- Results getImports(final UUID applicationId, final String cursor);
+ Results getImports(final UUID applicationId, final String ql, final String cursor);
/**
* Get the import
@@ -57,34 +58,38 @@ public interface ImportService {
/**
* Get the results
*
+ * @param applicationId The applicationId
* @param importId The import id to get files from
- * @param cursor The cursor used in parsing
- * @return
+ * @param ql The query executed (nullable)
+ * @param cursor The cursor passed (nullable)
*/
- Results getFileImports(final UUID applicationId, final UUID importId, final String cursor);
+ Results getFileImports(final UUID applicationId, final UUID importId, final String ql, final String cursor);
/**
- * Get the file import
- * @param importId
- * @param fileImportId
- * @return
+ * Get the results
+ *
+ * @param applicationId The applicationId
+ * @param importId The import id to get files from
+ *
+ * @return The FileImport
*/
FileImport getFileImport(final UUID applicationId, final UUID importId, final UUID fileImportId);
-
/**
* Get the results of failed imports
*
+ *
+ * @param applicationId The applicationId
* @param importId The import id to get files from
- * @param fileImportId the fileImportId
- * @param cursor The cursor used in parsing
- * @return
+ * @param ql The query executed (nullable)
+ * @param cursor The cursor passed (nullable)
*/
- Results getFailedImportEntities(final UUID applicationId, final UUID importId, final UUID fileImportId, final String cursor);
+ Results getFailedImportEntities(final UUID applicationId, final UUID importId, final UUID fileImportId, final String ql, final String cursor);
/**
* Get the failedimport entity from it's parentId
+ * @param applicationId
* @param importId
* @param fileImportId
* @param failedImportId
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/db8184c8/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
index ba72857..f35e6f2 100644
--- a/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/management/importer/ImportServiceImpl.java
@@ -45,6 +45,7 @@ import rx.functions.Func1;
import rx.functions.Func2;
import rx.schedulers.Schedulers;
+import javax.annotation.Nullable;
import javax.annotation.PostConstruct;
import java.io.File;
import java.util.*;
@@ -59,9 +60,9 @@ public class ImportServiceImpl implements ImportService {
public static final int HEARTBEAT_COUNT = 50;
public static final String APP_IMPORT_CONNECTION ="imports";
+ public static final String IMPORT_FILE_INCLUDES_CONNECTION = "includes";
private static final Logger logger = LoggerFactory.getLogger(ImportServiceImpl.class);
- public static final String IMPORT_FILE_INCLUDES = "includes";
int MAX_FILE_IMPORTS = 1000; // max number of file import jobs / import job
@@ -137,17 +138,17 @@ public class ImportServiceImpl implements ImportService {
@Override
- public Results getImports( final UUID applicationId, final String cursor ) {
+ public Results getImports( final UUID applicationId, @Nullable final String ql, @Nullable final String cursor ) {
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+
try {
final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
final Entity applicationEntity = getApplicationEntity( rootEm, applicationId );
- Query query = new Query();
- if ( cursor != null ) {
- query.setCursor( cursor );
- }
+ Query query = Query.fromQLNullSafe( ql );
+ query.setCursor( cursor );
//set our entity type
query.setEntityType( Schema.getDefaultSchema().getEntityType( Import.class ) );
@@ -162,6 +163,9 @@ public class ImportServiceImpl implements ImportService {
@Override
public Import getImport( final UUID applicationId, final UUID importId ) {
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+ Preconditions.checkNotNull( importId, "importId must be specified" );
+
try {
final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
@@ -181,8 +185,6 @@ public class ImportServiceImpl implements ImportService {
}
-
-
private Entity getApplicationEntity(final EntityManager rootEm, final UUID applicationId) throws Exception {
final Entity entity = rootEm.get( new SimpleEntityRef( "application_info", applicationId ) );
@@ -194,27 +196,115 @@ public class ImportServiceImpl implements ImportService {
}
@Override
- public Results getFileImports(final UUID applicationId, final UUID importId, final String cursor ) {
- return null;
+ public Results getFileImports(final UUID applicationId, final UUID importId, @Nullable final String ql, @Nullable final String cursor ) {
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+ Preconditions.checkNotNull( importId, "importId must be specified" );
+
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
+ final Import importEntity = getImport( applicationId, importId );
+
+ Query query = Query.fromQLNullSafe( ql );
+ query.setCursor( cursor );
+
+ //set our entity type
+ query.setEntityType( Schema.getDefaultSchema().getEntityType( Import.class ) );
+
+ return rootEm.searchCollection( importEntity, IMPORT_FILE_INCLUDES_CONNECTION, query );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to get import entity", e );
+ }
+
}
@Override
public FileImport getFileImport(final UUID applicationId, final UUID importId, final UUID fileImportId ) {
- return null;
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
+ final Import importEntity = getImport( applicationId, importId );
+
+ if ( importEntity == null ) {
+ throw new EntityNotFoundException( "Import not found with id " + importId );
+ }
+
+
+ final FileImport fileImport = rootEm.get( importId, FileImport.class );
+
+
+ // check if it's on the path
+ if ( !rootEm.isConnectionMember( importEntity, APP_IMPORT_CONNECTION, fileImport ) ) {
+ return null;
+ }
+
+ return fileImport;
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to load file import", e );
+ }
}
@Override
- public Results getFailedImportEntities(final UUID applicationId, final UUID importId, final UUID fileImportId, final String cursor ) {
- return null;
+ public Results getFailedImportEntities(final UUID applicationId, final UUID importId, final UUID fileImportId, @Nullable final String ql, @Nullable final String cursor ) {
+
+ Preconditions.checkNotNull( applicationId, "applicationId must be specified" );
+ Preconditions.checkNotNull( importId, "importId must be specified" );
+ Preconditions.checkNotNull( fileImportId, "fileImportId must be specified" );
+
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
+ final FileImport importEntity = getFileImport( applicationId, importId, fileImportId );
+
+ Query query = Query.fromQLNullSafe( ql );
+ query.setCursor( cursor );
+
+ //set our entity type
+ query.setEntityType( Schema.getDefaultSchema().getEntityType( FailedImportEntity.class ) );
+
+ return rootEm.searchCollection( importEntity, FileImportTracker.ERRORS_CONNECTION_NAME, query );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to get import entity", e );
+ }
}
@Override
public FailedImportEntity getFailedImportEntity(final UUID applicationId, final UUID importId, final UUID fileImportId,
final UUID failedImportId ) {
- return null;
+ try {
+ final EntityManager rootEm = emf.getEntityManager( emf.getManagementAppId() );
+
+
+ final FileImport importEntity = getFileImport( applicationId, importId, fileImportId );
+
+ if ( importEntity == null ) {
+ throw new EntityNotFoundException( "Import not found with id " + importId );
+ }
+
+
+ final FailedImportEntity fileImport = rootEm.get( importId, FailedImportEntity.class );
+
+
+ // check if it's on the path
+ if ( !rootEm.isConnectionMember( importEntity, FileImportTracker.ERRORS_CONNECTION_NAME, fileImport ) ) {
+ return null;
+ }
+
+ return fileImport;
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to load file import", e );
+ }
}
@@ -248,7 +338,7 @@ public class ImportServiceImpl implements ImportService {
try {
// create a connection between the main import job and the sub FileImport Job
- emManagementApp.createConnection(importEntity, IMPORT_FILE_INCLUDES, fileImport);
+ emManagementApp.createConnection(importEntity, IMPORT_FILE_INCLUDES_CONNECTION, fileImport);
logger.debug("Created connection from {}:{} to {}:{}",
new Object[] {
@@ -284,11 +374,12 @@ public class ImportServiceImpl implements ImportService {
try {
EntityManager emMgmtApp = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
- Query query = Query.fromQL("select *");
+ Query query = Query.fromQL( "select *" );
query.setEntityType("file_import");
- query.setConnectionType( IMPORT_FILE_INCLUDES );
+ query.setConnectionType( IMPORT_FILE_INCLUDES_CONNECTION );
query.setLimit(MAX_FILE_IMPORTS);
+ //TODO, this won't work with more than 100 files
Results entities = emMgmtApp.searchConnectedEntities( importRoot, query );
return entities.size();
@@ -638,7 +729,7 @@ public class ImportServiceImpl implements ImportService {
// get parent import job of this file import job
Results importJobResults =
- emManagementApp.getConnectingEntities( fileImport, IMPORT_FILE_INCLUDES, null, Level.ALL_PROPERTIES );
+ emManagementApp.getConnectingEntities( fileImport, IMPORT_FILE_INCLUDES_CONNECTION, null, Level.ALL_PROPERTIES );
List<Entity> importEntities = importJobResults.getEntities();
UUID importId = importEntities.get( 0 ).getUuid();
Import importEntity = emManagementApp.get( importId, Import.class );
@@ -649,7 +740,7 @@ public class ImportServiceImpl implements ImportService {
EntityManager emMgmtApp = emf.getEntityManager( CpNamingUtils.MANAGEMENT_APPLICATION_ID );
Query query = Query.fromQL("select *");
query.setEntityType("file_import");
- query.setConnectionType( IMPORT_FILE_INCLUDES );
+ query.setConnectionType( IMPORT_FILE_INCLUDES_CONNECTION );
query.setLimit(MAX_FILE_IMPORTS);
Results entities = emMgmtApp.searchConnectedEntities(importEntity, query);