You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/05/27 01:06:20 UTC
[01/12] incubator-usergrid git commit: Refactor of pipeline to
support type mapping for clarity
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-608 29a4009eb -> 87963740a
Refactor of pipeline to support type mapping for clarity
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3a1784f0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3a1784f0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3a1784f0
Branch: refs/heads/USERGRID-608
Commit: 3a1784f0455acae20c7dfbde61e9493d572ad549
Parents: 5b1dfa1
Author: Todd Nine <tn...@apigee.com>
Authored: Tue May 19 17:05:04 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue May 19 18:07:47 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 32 +-
.../corepersistence/CpEntityManagerFactory.java | 13 +-
.../corepersistence/CpRelationManager.java | 158 +++++-----
.../corepersistence/pipeline/Pipeline.java | 121 --------
.../pipeline/PipelineBuilderFactory.java | 39 ---
.../pipeline/PipelineModule.java | 9 -
.../pipeline/PipelineOperations.java | 30 ++
.../corepersistence/pipeline/read/Filter.java | 9 +-
.../pipeline/read/FilterFactory.java | 5 +-
.../pipeline/read/FilterPipeline.java | 132 +++++++++
.../pipeline/read/ReadPipelineBuilder.java | 104 -------
.../pipeline/read/ReadPipelineBuilderImpl.java | 296 -------------------
.../pipeline/read/collect/EntityFilter.java | 68 -----
.../read/collect/EntityResumeFilter.java | 68 +++++
.../read/elasticsearch/CandidateIdFilter.java | 46 ++-
15 files changed, 380 insertions(+), 750 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 63018cb..7a56631 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -17,14 +17,29 @@ package org.apache.usergrid.corepersistence;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.pipeline.PipelineBuilderFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -165,7 +180,8 @@ public class CpEntityManager implements EntityManager {
private final AsyncEventService indexService;
- private PipelineBuilderFactory pipelineBuilderFactory;
+ private final FilterFactory filterFactory;
+ private final CollectorFactory collectorFactory;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -207,7 +223,7 @@ public class CpEntityManager implements EntityManager {
*/
public CpEntityManager( final CassandraService cass, final CounterUtils counterUtils, final AsyncEventService indexService, final ManagerCache managerCache,
final MetricsFactory metricsFactory, final EntityManagerFig entityManagerFig,
- final PipelineBuilderFactory pipelineBuilderFactory , final UUID applicationId ) {
+ final FilterFactory filterFactory, final CollectorFactory collectorFactory, final UUID applicationId ) {
this.entityManagerFig = entityManagerFig;
@@ -216,8 +232,10 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
Preconditions.checkNotNull( indexService, "indexService must not be null" );
- Preconditions.checkNotNull( pipelineBuilderFactory, "pipelineBuilderFactory must not be null" );
- this.pipelineBuilderFactory = pipelineBuilderFactory;
+ Preconditions.checkNotNull( filterFactory, "filterFactory must not be null" );
+ Preconditions.checkNotNull( collectorFactory, "collectorFactory must not be null" );
+ this.filterFactory = filterFactory;
+ this.collectorFactory = collectorFactory;
this.managerCache = managerCache;
@@ -732,7 +750,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager relationManager =
- new CpRelationManager( metricsFactory, managerCache, pipelineBuilderFactory, indexService, this, entityManagerFig, applicationId, entityRef );
+ new CpRelationManager( metricsFactory, managerCache, filterFactory, collectorFactory, indexService, this, entityManagerFig, applicationId, entityRef );
return relationManager;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 63e2869..5055538 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -35,7 +35,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.ReIndexService;
-import org.apache.usergrid.corepersistence.pipeline.PipelineBuilderFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
import org.apache.usergrid.persistence.AbstractEntity;
@@ -125,7 +126,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
private final AsyncEventService indexService;
- private final PipelineBuilderFactory pipelineBuilderFactory;
+ private final FilterFactory filterFactory;
+ private final CollectorFactory collectorFactory;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector ) {
@@ -139,7 +141,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
this.indexService = injector.getInstance( AsyncEventService.class );
- this.pipelineBuilderFactory = injector.getInstance( PipelineBuilderFactory.class );
+ this.filterFactory = injector.getInstance( FilterFactory.class );
+ this.collectorFactory = injector.getInstance( CollectorFactory.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
@@ -198,7 +201,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, entityManagerFig, pipelineBuilderFactory, applicationId );
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, entityManagerFig,
+
+ filterFactory, collectorFactory, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 4993d88..6201fe8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -17,16 +17,23 @@
package org.apache.usergrid.corepersistence;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.UUID;
-import org.apache.usergrid.persistence.graph.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.pipeline.PipelineBuilderFactory;
-import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterPipeline;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.corepersistence.results.ObservableQueryExecutor;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
@@ -49,6 +56,10 @@ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
@@ -63,7 +74,6 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.MapUtils;
-import com.codahale.metrics.Timer;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
@@ -97,7 +107,6 @@ public class CpRelationManager implements RelationManager {
private final EntityManagerFig entityManagerFig;
private ManagerCache managerCache;
- private final PipelineBuilderFactory pipelineBuilderFactory;
private EntityManager em;
@@ -111,13 +120,16 @@ public class CpRelationManager implements RelationManager {
private final AsyncEventService indexService;
- private MetricsFactory metricsFactory;
- private Timer updateCollectionTimer;
+
+ private final FilterFactory filterFactory;
+ private final CollectorFactory collectorFactory;
+
public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache,
- final PipelineBuilderFactory pipelineBuilderFactory, final AsyncEventService indexService,
- final EntityManager em, final EntityManagerFig entityManagerFig, final UUID applicationId, final EntityRef headEntity) {
+ final FilterFactory filterFactory, final CollectorFactory collectorFactory, final AsyncEventService indexService,
+ final EntityManager em, final EntityManagerFig entityManagerFig, final UUID applicationId,
+ final EntityRef headEntity ) {
Assert.notNull( em, "Entity manager cannot be null" );
@@ -134,11 +146,9 @@ public class CpRelationManager implements RelationManager {
this.headEntity = headEntity;
this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
- this.pipelineBuilderFactory = pipelineBuilderFactory;
- this.metricsFactory = metricsFactory;
- this.updateCollectionTimer =
- metricsFactory.getTimer( CpRelationManager.class, "relation.manager.es.update.collection" );
+ this.filterFactory = filterFactory;
+ this.collectorFactory = collectorFactory;
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading head entity {}:{} from app {}", new Object[] {
@@ -162,7 +172,7 @@ public class CpRelationManager implements RelationManager {
public Set<String> getCollectionIndexes( String collectionName ) throws Exception {
GraphManager gm = managerCache.getGraphManager( applicationScope );
- String edgeTypePrefix = CpNamingUtils.getEdgeTypeFromCollectionName(collectionName);
+ String edgeTypePrefix = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
logger.debug( "getCollectionIndexes(): Searching for edge type prefix {} to target {}:{}", new Object[] {
edgeTypePrefix, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid()
@@ -198,7 +208,7 @@ public class CpRelationManager implements RelationManager {
private Map<EntityRef, Set<String>> getContainers() {
- return getContainers(-1, null, null);
+ return getContainers( -1, null, null );
}
@@ -215,14 +225,14 @@ public class CpRelationManager implements RelationManager {
Observable<Edge> edges =
gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeType, null ) )
- .flatMap(new Func1<String, Observable<Edge>>() {
+ .flatMap( new Func1<String, Observable<Edge>>() {
@Override
- public Observable<Edge> call(final String edgeType) {
+ public Observable<Edge> call( final String edgeType ) {
return gm.loadEdgesToTarget(
- new SimpleSearchByEdgeType(cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent()));
+ new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
- });
+ } );
//if our limit is set, take them. Note this logic is still borked, we can't possibly fit everything in memmory
if ( limit > -1 ) {
@@ -250,7 +260,7 @@ public class CpRelationManager implements RelationManager {
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
- String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType(connectionType);
+ String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType( connectionType );
logger.debug( "isConnectionMember(): Checking for edge type {} from {}:{} to {}:{}", new Object[] {
edgeType, headEntity.getType(), headEntity.getUuid(), entity.getType(), entity.getUuid()
@@ -271,13 +281,13 @@ public class CpRelationManager implements RelationManager {
Id entityId = new SimpleId( entity.getUuid(), entity.getType() );
- String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName(collectionName);
+ String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
logger.debug( "isCollectionMember(): Checking for edge type {} from {}:{} to {}:{}", new Object[] {
edgeType, headEntity.getType(), headEntity.getUuid(), entity.getType(), entity.getUuid()
} );
- GraphManager gm = managerCache.getGraphManager(applicationScope);
+ GraphManager gm = managerCache.getGraphManager( applicationScope );
Observable<Edge> edges = gm.loadEdgeVersions(
new SimpleSearchByEdge( new SimpleId( headEntity.getUuid(), headEntity.getType() ), edgeType, entityId,
Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
@@ -343,7 +353,8 @@ public class CpRelationManager implements RelationManager {
return null;
}
- return addToCollection( collectionName, itemRef, ( collection != null && collection.getLinkedCollection() != null ) );
+ return addToCollection( collectionName, itemRef,
+ ( collection != null && collection.getLinkedCollection() != null ) );
}
@@ -403,7 +414,7 @@ public class CpRelationManager implements RelationManager {
logger.debug( "Wrote edge {}", edge );
}
- indexService.queueNewEdge(applicationScope, memberEntity, edge);
+ indexService.queueNewEdge( applicationScope, memberEntity, edge );
if ( logger.isDebugEnabled() ) {
@@ -518,7 +529,8 @@ public class CpRelationManager implements RelationManager {
//run our delete
- final Edge collectionToItemEdge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
+ final Edge collectionToItemEdge =
+ createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
gm.markEdge( collectionToItemEdge ).toBlocking().last();
@@ -575,7 +587,7 @@ public class CpRelationManager implements RelationManager {
results = em.getCollection( headEntity, srcRelationName, null, 5000, Level.REFS, false );
}
else {
- results = em.getTargetEntities(headEntity, srcRelationName, null, Level.REFS);
+ results = em.getTargetEntities( headEntity, srcRelationName, null, Level.REFS );
}
if ( ( results != null ) && ( results.size() > 0 ) ) {
@@ -617,50 +629,54 @@ public class CpRelationManager implements RelationManager {
query = adjustQuery( query );
+ final FilterPipeline<Id> filterPipeline = new FilterPipeline( applicationScope, query.getCursor(), query.getLimit() ).withFilter( filterFactory.getEntityIdFilter( cpHeadEntity.getId() ) );
- final ReadPipelineBuilder readPipelineBuilder =
- pipelineBuilderFactory.createReadPipelineBuilder(applicationScope);
- //set our fields applicable to both operations
- readPipelineBuilder.withCursor(query.getCursor());
- readPipelineBuilder.withLimit( Optional.of(query.getLimit()));
-
- //TODO, this should be removed when the CP relation manager is removed
- readPipelineBuilder.setStartId( cpHeadEntity.getId() );
+ final FilterPipeline<org.apache.usergrid.persistence.model.entity.Entity> entityFilterPipeline;
if ( query.isGraphSearch() ) {
- readPipelineBuilder.getCollection( collectionName );
+ entityFilterPipeline = filterPipeline.withFilter( filterFactory.readGraphCollectionFilter( collectionName ) )
+ .withFilter( filterFactory.entityLoadFilter() );
}
else {
final String entityType = collection.getType();
- readPipelineBuilder.getCollectionWithQuery( collectionName, entityType, query.getQl().get() );
+ entityFilterPipeline = filterPipeline.withFilter(
+ filterFactory.elasticSearchCollectionFilter( query.getQl().get(), collectionName, entityType ) )
+ .withFilter( filterFactory.candidateEntityFilter() );
}
- final Observable<ResultsPage> resultsObservable = readPipelineBuilder.execute();
+ final Observable<ResultsPage> resultsObservable =
+ entityFilterPipeline.withFilter( filterFactory.entityResumeFilter() )
+ .withCollector( collectorFactory.getResultsPageCollector() ).execute();
return new ObservableQueryExecutor( resultsObservable ).next();
}
+
@Override
- public Results searchCollectionConsistent( String collectionName, Query query, int expectedResults ) throws Exception {
+ public Results searchCollectionConsistent( String collectionName, Query query, int expectedResults )
+ throws Exception {
Results results;
long maxLength = entityManagerFig.pollForRecordsTimeout();
long sleepTime = entityManagerFig.sleep();
boolean found;
long current = System.currentTimeMillis(), length = 0;
do {
- results = searchCollection(collectionName, query);
+ results = searchCollection( collectionName, query );
length = System.currentTimeMillis() - current;
found = expectedResults == results.size();
- if(found){
+ if ( found ) {
break;
}
- Thread.sleep(sleepTime);
- }while (!found && length <= maxLength);
- if(logger.isInfoEnabled()){
- logger.info(String.format("Consistent Search finished in %s, results=%s, expected=%s...dumping stack",length, results.size(),expectedResults));
+ Thread.sleep( sleepTime );
+ }
+ while ( !found && length <= maxLength );
+ if ( logger.isInfoEnabled() ) {
+ logger.info( String
+ .format( "Consistent Search finished in %s, results=%s, expected=%s...dumping stack", length,
+ results.size(), expectedResults ) );
Thread.dumpStack();
}
return results;
@@ -836,7 +852,7 @@ public class CpRelationManager implements RelationManager {
@Override
- public Results getTargetEntities(String connectionType, String connectedEntityType, Level level)
+ public Results getTargetEntities( String connectionType, String connectedEntityType, Level level )
throws Exception {
//until this is refactored properly, we will delegate to a search by query
@@ -849,20 +865,19 @@ public class CpRelationManager implements RelationManager {
query.setEntityType( connectedEntityType );
query.setResultsLevel( level );
- return searchTargetEntities(query);
+ return searchTargetEntities( query );
}
@Override
- public Results getSourceEntities(String connType, String fromEntityType, Level resultsLevel)
- throws Exception {
+ public Results getSourceEntities( String connType, String fromEntityType, Level resultsLevel ) throws Exception {
- return getSourceEntities(connType, fromEntityType, resultsLevel, -1);
+ return getSourceEntities( connType, fromEntityType, resultsLevel, -1 );
}
@Override
- public Results getSourceEntities(String connType, String fromEntityType, Level level, int count)
+ public Results getSourceEntities( String connType, String fromEntityType, Level level, int count )
throws Exception {
// looking for edges to the head entity
@@ -895,7 +910,7 @@ public class CpRelationManager implements RelationManager {
@Override
- public Results searchTargetEntities(Query query) throws Exception {
+ public Results searchTargetEntities( Query query ) throws Exception {
Preconditions.checkNotNull( query, "query cannot be null" );
@@ -909,37 +924,40 @@ public class CpRelationManager implements RelationManager {
query = adjustQuery( query );
final String entityType = query.getEntityType();
- //set startid -- graph | es query filter -- load entities filter (verifies exists) --> results page collector -> 1.0 results
+ //set startid -- graph | es query filter -- load entities filter (verifies exists) --> results page collector
+ // -> 1.0 results
// startid -- graph edge load -- entity load (verify) from ids -> results page collector
// startid -- eq query candiddate -- entity load (verify) from canddiates -> results page collector
//startid -- graph edge load -- entity id verify --> filter to connection ref --> connection ref collector
- //startid -- eq query candiddate -- candidate id verify --> filter to connection ref --> connection ref collector
+ //startid -- eq query candiddate -- candidate id verify --> filter to connection ref --> connection ref
+ // collector
+
- final ReadPipelineBuilder readPipelineBuilder =
- pipelineBuilderFactory.createReadPipelineBuilder(applicationScope);
- //readPipelineBuilder.startId().load().collect()
+ final FilterPipeline<Id> filterPipeline =
+ new FilterPipeline( applicationScope, query.getCursor(), query.getLimit() )
+ .withFilter( filterFactory.getEntityIdFilter( cpHeadEntity.getId() ) );
- //set our fields applicable to both operations
- readPipelineBuilder
- .withCursor(query.getCursor())
- .withLimit(Optional.of(query.getLimit()))
- //TODO, this should be removed when the CP relation manager is removed
- .setStartId( cpHeadEntity.getId() );
+
+ final FilterPipeline<org.apache.usergrid.persistence.model.entity.Entity> entityFilterPipeline;
if ( query.isGraphSearch() ) {
- // if(query.getResultsLevel() == Level.ALL_PROPERTIES)
- readPipelineBuilder.getConnection( connection );
- //else
+ entityFilterPipeline = filterPipeline.withFilter( filterFactory.readGraphConnectionFilter( connection ) )
+ .withFilter( filterFactory.entityLoadFilter() );
}
+
else {
- readPipelineBuilder.getConnectionWithQuery( connection, Optional.fromNullable( entityType ),
- query.getQl().get() );
+
+ entityFilterPipeline = filterPipeline.withFilter( filterFactory
+ .elasticSearchConnectionFilter( query.getQl().get(), connection, Optional.fromNullable( entityType ) ) )
+ .withFilter( filterFactory.candidateEntityFilter() );
}
- final Observable<ResultsPage> resultsObservable = readPipelineBuilder.execute();
+ final Observable<ResultsPage> resultsObservable =
+ entityFilterPipeline.withFilter( filterFactory.entityResumeFilter() )
+ .withCollector( collectorFactory.getResultsPageCollector() ).execute();
return new ObservableQueryExecutor( resultsObservable ).next();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java
deleted file mode 100644
index 26cf346..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline;
-
-
-import java.util.List;
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
-import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
-import org.apache.usergrid.corepersistence.pipeline.read.Collector;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * A pipeline that will allow us to build a traversal command for execution
- *
- * See http://martinfowler.com/articles/collection-pipeline/ for some examples
- *
- * TODO: Re work the cursor and limit phases. They need to be lazily evaluated, not added on build time
- */
-public class Pipeline<R> {
-
-
- private final ApplicationScope applicationScope;
- private final List<PipelineOperation> idPipelineOperationList;
- private final Collector<?, R> collector;
- private final RequestCursor requestCursor;
-
- private final int limit;
-
-
- private int idCount = 0;
-
-
- /**
- * Our first pass, where we implement our start point as an Id until we can use this to perform our entire
- * traversal. Eventually as we untangle the existing Query service nightmare, the sourceId will be remove and
- * should only be traversed from the root application
- */
- public Pipeline( final ApplicationScope applicationScope, final List<PipelineOperation> pipelineOperations,
- final Collector<?, R> collector, final Optional<String> cursor, final int limit ) {
-
- this.applicationScope = applicationScope;
- this.idPipelineOperationList = pipelineOperations;
- this.collector = collector;
- this.limit = limit;
-
- this.requestCursor = new RequestCursor( cursor );
- }
-
-
- /**
- * Execute the pipline construction, returning an observable of results
- * @return
- */
- public Observable<R> execute(){
-
-
- Observable traverseObservable = Observable.just( new FilterResult<>( applicationScope.getApplication(), Optional.absent() ));
-
- //build our traversal commands
- for ( PipelineOperation pipelineOperation : idPipelineOperationList ) {
- setState( pipelineOperation );
-
- //TODO, see if we can wrap this observable in our ObservableTimer so we can see how long each filter takes
-
-
- traverseObservable = traverseObservable.compose( pipelineOperation );
- }
-
-
- setState( collector );
-
- final Observable<R> response = traverseObservable.compose( collector );
-
-
- //append the optional cursor into the response for the caller to use
- return response;
- }
-
-
-
-
- /**
- * Set the id of the state
- */
- private void setState( final PipelineOperation pipelineOperation ) {
-
-
- final PipelineContext context = new PipelineContext( applicationScope, requestCursor,
- limit, idCount );
-
- pipelineOperation.setContext( context );
-
- //done for clarity
- idCount++;
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
deleted file mode 100644
index 9916bc1..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline;
-
-
-import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-
-/**
- * Factory for creating pipeline builders
- */
-public interface PipelineBuilderFactory {
-
-
- /**
- * Create a read pipeline builder
- * @param applicationScope
- * @return
- */
- ReadPipelineBuilder createReadPipelineBuilder( final ApplicationScope applicationScope );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
index 3018718..ef696bd 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
@@ -22,9 +22,6 @@ package org.apache.usergrid.corepersistence.pipeline;
import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
-import org.apache.usergrid.corepersistence.pipeline.read.ReadFilterFactoryImpl;
-import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
-import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilderImpl;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -42,12 +39,6 @@ public class PipelineModule extends AbstractModule {
// bind( FilterFactory.class ).to( ReadFilterFactoryImpl.class );
- //Use Guice to create the builder since we don't really need to do anything
- //other than DI when creating the filters
- install( new FactoryModuleBuilder().implement( ReadPipelineBuilder.class, ReadPipelineBuilderImpl.class )
- .build( PipelineBuilderFactory.class ) );
-
-
// install( new Factory)
//Use Guice to create the builder since we don't really need to do anything
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperations.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperations.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperations.java
new file mode 100644
index 0000000..3929a97
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperations.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+public interface PipelineOperations {
+
+ /**
+ * Add the pipeline operation to the set of operations
+ * @param po
+ */
+ void add( PipelineOperation po );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
index 054a85a..ee01602 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
@@ -28,4 +28,11 @@ import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
* an observable of FilterResults. Filters should never emit groups or objects that represent collections. Items should
* always be emitted 1 at a time. It is the responsibility of the collector to aggregate results.
*/
-public interface Filter<T, R> extends PipelineOperation<T, FilterResult<R>> {}
+public interface Filter<T, R> extends PipelineOperation<T, FilterResult<R>> {
+
+ /**
+ * Get the builder for the next phase
+ * @return
+ */
+// B getNextBuilder();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
index a2f1605..d297c2a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
@@ -20,8 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read;
-import org.apache.usergrid.corepersistence.pipeline.read.collect.EntityFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.collect.IdCursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.EntityResumeFilter;
import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.CandidateEntityFilter;
import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.CandidateIdFilter;
import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.ElasticSearchCollectionFilter;
@@ -139,5 +138,5 @@ public interface FilterFactory {
* Create a new instance of our entity filter
* @return
*/
- EntityFilter entityFilter();
+ EntityResumeFilter entityResumeFilter();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java
new file mode 100644
index 0000000..f8bbdd8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.corepersistence.pipeline.PipelineContext;
+import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+import rx.Observable;
+
+
+/**
+ * Pipeline for applying our UG domain specific filters.
+ *
+ * Modeled after an observable, with typing to allow input of specific filters
+ *
+ * @param InputType the input type in the current pipeline state
+ */
+public class FilterPipeline<InputType> {
+
+
+ private int idCount = 0;
+
+ private final ApplicationScope applicationScope;
+
+
+ private final RequestCursor requestCursor;
+ private int limit;
+
+ //Generics hell, intentionally without a generic, we check at the filter level
+ private Observable currentObservable;
+
+
+ /**
+ * Create our filter pipeline
+ */
+ public FilterPipeline( final ApplicationScope applicationScope, final Optional<String> cursor, final int limit ) {
+
+
+ ValidationUtils.validateApplicationScope( applicationScope );
+ Preconditions.checkNotNull( cursor, "cursor optional is required" );
+ Preconditions.checkArgument( limit > 0, "limit must be > 0" );
+
+
+ this.applicationScope = applicationScope;
+
+ //init our cursor to empty
+ this.requestCursor = new RequestCursor( cursor );
+
+ //set the default limit
+ this.limit = limit;
+
+ //set our observable to start at the application
+ final FilterResult<Id> filter = new FilterResult<>( applicationScope.getApplication(), Optional.absent() );
+ this.currentObservable = Observable.just( filter );
+ }
+
+
+ public <OutputType> FilterPipeline<OutputType> withFilter(
+ final Filter<? super InputType, ? extends OutputType> filter ) {
+
+
+ setUp( filter );
+
+ return ( FilterPipeline<OutputType> ) this;
+ }
+
+
+ public <OutputType> FilterPipeline<OutputType> withCollector(
+ final Collector<? super InputType, ? extends OutputType> collector ) {
+
+
+ setUp( collector );
+
+ return ( FilterPipeline<OutputType> ) this;
+ }
+
+
+ private <OutputType> void setUp(
+ final PipelineOperation<? super InputType, ? extends OutputType> pipelineOperation ) {
+ setState( pipelineOperation );
+
+ currentObservable = currentObservable.compose( pipelineOperation );
+ }
+
+
+ /**
+ * Return the observable of the filter pipeline
+ */
+ public Observable<InputType> execute() {
+ return currentObservable;
+ }
+
+
+ /**
+ * Set the id of the state
+ */
+ private void setState( final PipelineOperation pipelineOperation ) {
+
+
+ final PipelineContext context = new PipelineContext( applicationScope, requestCursor, limit, idCount );
+
+ pipelineOperation.setContext( context );
+
+ //done for clarity
+ idCount++;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
deleted file mode 100644
index d0e87b3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * An instance of a pipeline builder for building commands on our read pipline
- *
- * Each invocation of the method will assemble the underlying pipe and updating it's state
- *
- * Results are added by invoking execute.
- */
-public interface ReadPipelineBuilder {
-
-
- /**
- * Set the cursor
- * @param cursor
- */
- ReadPipelineBuilder withCursor(final Optional<String> cursor);
-
- /**
- * Set the limit of our page sizes
- * @param limit
- * @return
- */
- ReadPipelineBuilder withLimit(final Optional<Integer> limit);
-
- /**
- * An operation to bridge 2.0-> 1.0. Should be removed when everyone uses the pipeline
- * @param id
- * @return
- */
- ReadPipelineBuilder setStartId(final Id id);
-
-
- /**
- * Add a get entity to the pipeline
- */
- ReadPipelineBuilder getEntityViaCollection( final String collectionName, final Id entityId );
-
-
- /**
- * Add get Collection from our previous source
- */
- ReadPipelineBuilder getCollection( final String collectionName );
-
- /**
- * Get all entities with a query
- */
- ReadPipelineBuilder getCollectionWithQuery( final String collectionName,final String entityType, final String query);
-
- /**
- * Get an entity via the connection name and entity Id
- */
- ReadPipelineBuilder getEntityViaConnection( final String connectionName, final Id entityId );
-
- /**
- * Get all entities in a connection by the connection name
- */
- ReadPipelineBuilder getConnection( final String connectionName );
-
- /**
- * Get all entities in a connection of the specified connection type
- */
- ReadPipelineBuilder getConnection( final String connectionName, final String entityType );
-
- /**
- * Get all entities in a connection with a query and a target entity type
- */
- ReadPipelineBuilder getConnectionWithQuery( final String connectionName, final Optional<String> entityType,
- final String query );
-
-
- /**
- * Load our entity results when our previous filter calls graph
- * @return
- */
- Observable<ResultsPage> execute();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
deleted file mode 100644
index 28446ad..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.usergrid.corepersistence.pipeline.Pipeline;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.CandidateEntityFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.EntityLoadFilter;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import rx.Observable;
-
-
-/**
- * An implementation of our builder for piplines
- */
-public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
-
- private static final int DEFAULT_LIMIT = 10;
-
- private final FilterFactory filterFactory;
-
- private final CollectorState collectorState;
-
- private final ApplicationScope applicationScope;
-
- private final CollectorFactory collectorFactory;
-
-
- /**
- * Our pointer to our collect filter. Set or cleared with each operation that's performed so the correct results are
- * rendered
- */
- private List<Filter> filters;
-
-
- private Optional<String> cursor;
- private int limit;
-
-
- @Inject
- public ReadPipelineBuilderImpl( final FilterFactory filterFactory, final CollectorFactory collectorFactory,
- @Assisted final ApplicationScope applicationScope ) {
- this.filterFactory = filterFactory;
-
- this.applicationScope = applicationScope;
- this.collectorFactory = collectorFactory;
-
- //init our cursor to empty
- this.cursor = Optional.absent();
-
- //set the default limit
- this.limit = DEFAULT_LIMIT;
-
-
- this.collectorState = new CollectorState( );
-
- this.filters = new ArrayList<>();
- }
-
-
- @Override
- public ReadPipelineBuilder withCursor( final Optional<String> cursor ) {
- Preconditions.checkNotNull( cursor, "cursor must not be null" );
- this.cursor = cursor;
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder withLimit( final Optional<Integer> limit ) {
- Preconditions.checkNotNull( limit, "limit must not be null" );
- this.limit = limit.or( DEFAULT_LIMIT );
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder setStartId( final Id id ) {
- ValidationUtils.verifyIdentity( id );
-
- filters.add( filterFactory.getEntityIdFilter( id ) );
-
- this.collectorState.clear();
-
-
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder getEntityViaCollection( final String collectionName, final Id entityId ) {
- Preconditions.checkNotNull( collectionName, "collectionName must not be null" );
- ValidationUtils.verifyIdentity( entityId );
-
- filters.add( filterFactory.readGraphCollectionByIdFilter( collectionName, entityId ) );
-
- this.collectorState.setIdEntityLoaderFilter();
-
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder getCollection( final String collectionName ) {
- Preconditions.checkNotNull( collectionName, "collectionName must not be null" );
-
- filters.add( filterFactory.readGraphCollectionFilter( collectionName ) );
-
- this.collectorState.setIdEntityLoaderFilter();
-
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String entityType, final String query ) {
- Preconditions.checkNotNull( collectionName, "collectionName must not be null" );
- Preconditions.checkNotNull( query, "query must not be null" );
-
- //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
-
- filters.add( filterFactory.elasticSearchCollectionFilter( query, collectionName, entityType ) );
-
- this.collectorState.setCandidateEntityFilter();
-
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder getEntityViaConnection( final String connectionName, final Id entityId ) {
- Preconditions.checkNotNull( connectionName, "connectionName must not be null" );
- ValidationUtils.verifyIdentity( entityId );
-
- filters.add( filterFactory.readGraphConnectionByIdFilter( connectionName, entityId ) );
- collectorState.setIdEntityLoaderFilter();
-
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder getConnection( final String connectionName ) {
- Preconditions.checkNotNull( connectionName, "connectionName must not be null" );
- filters.add( filterFactory.readGraphConnectionFilter( connectionName ) );
- collectorState.setIdEntityLoaderFilter();
-
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder getConnection( final String connectionName, final String entityType ) {
- Preconditions.checkNotNull( connectionName, "connectionName must not be null" );
- Preconditions.checkNotNull( connectionName, "entityType must not be null" );
-
- filters.add( filterFactory.readGraphConnectionByTypeFilter( connectionName, entityType ) );
-
- collectorState.setIdEntityLoaderFilter();
- return this;
- }
-
-
- @Override
- public ReadPipelineBuilder getConnectionWithQuery( final String connectionName, final Optional<String> entityType,
- final String query ) {
-
- Preconditions.checkNotNull( connectionName, "connectionName must not be null" );
- Preconditions.checkNotNull( connectionName, "entityType must not be null" );
- Preconditions.checkNotNull( query, "query must not be null" );
-
- filters.add( filterFactory.elasticSearchConnectionFilter( query, connectionName, entityType ) );
- collectorState.setCandidateEntityFilter();
- return this;
- }
-
-
- @Override
- public Observable<ResultsPage> execute() {
-
- ValidationUtils.validateApplicationScope( applicationScope );
-
-
- //add our last filter that will generate entities
- final Filter<?, Entity> entityLoadFilter = collectorState.getFinalFilter();
-
- filters.add( entityLoadFilter );
-
- //add the filter that skips the first result on resume
- final Filter<Entity, Entity> cursorEntityFilter = filterFactory.entityFilter();
-
- filters.add( cursorEntityFilter );
-
-
- //execute our collector
- final Collector<?, ResultsPage> collector = collectorFactory.getResultsPageCollector();
-
- Preconditions.checkNotNull( collector,
- "You have not specified an operation that creates a collection filter. This is required for loading "
- + "results" );
-
-
- Preconditions.checkNotNull( cursor, "A cursor should be initialized even if absent" );
-
- Preconditions.checkArgument( limit > 0, "limit must be > than 0" );
-
-
- Pipeline pipeline = new Pipeline( applicationScope, filters, collector, cursor, limit );
-
-
- return pipeline.execute();
- }
-
-
- /**
- * A mutable state for our collectors. Rather than create a new instance each time, we create a singleton
- * collector
- */
- private final class CollectorState {
-
-
- private EntityLoadFilter entityLoadCollector;
-
- private CandidateEntityFilter candidateEntityFilter;
-
- private Filter entityLoadFilter;
-
-
-
- private CollectorState( ){}
-
-
- /**
- * Set our final filter to be a load entity by Id filter
- */
- public void setIdEntityLoaderFilter() {
- if ( entityLoadCollector == null ) {
- entityLoadCollector = filterFactory.entityLoadFilter();
- }
-
-
- entityLoadFilter = entityLoadCollector;
- }
-
-
- /**
- * Set our final filter to be a load entity by candidate filter
- */
- public void setCandidateEntityFilter() {
- if ( candidateEntityFilter == null ) {
- candidateEntityFilter = filterFactory.candidateEntityFilter();
- }
-
- entityLoadFilter = candidateEntityFilter;
- }
-
-
- public void clear() {
- entityLoadFilter = null;
- }
-
-
- public Filter<?, Entity> getFinalFilter() {
- return entityLoadFilter;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityFilter.java
deleted file mode 100644
index daf2e7f..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityFilter.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.collect;
-
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * A filter that is used when we can potentially serialize pages via cursor. This will filter the first result, only if
- * it matches the Id that was set
- */
-public class EntityFilter extends AbstractPathFilter<Entity, Entity, Id> implements Filter<Entity, Entity> {
-
-
- @Override
- public Observable<FilterResult<Entity>> call( final Observable<FilterResult<Entity>> filterResultObservable ) {
-
- //filter only the first id, then map into our path for our next pass
-
-
- return filterResultObservable.skipWhile( filterResult -> {
-
- final Optional<Id> startFromCursor = getSeekValue();
-
- return startFromCursor.isPresent() && startFromCursor.get().equals( filterResult.getValue().getId() );
- } ).map( filterResult -> {
-
-
- final Entity entity = filterResult.getValue();
- final Id entityId = entity.getId();
-
- return createFilterResult( entity, entityId, filterResult.getPath() );
- } );
- }
-
-
- @Override
- protected CursorSerializer<Id> getCursorSerializer() {
- return IdCursorSerializer.INSTANCE;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java
new file mode 100644
index 0000000..2917b61
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.collect;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.Filter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * A filter that is used when we can potentially serialize pages via cursor. This will filter the first result, only if
+ * it matches the Id that was set
+ */
+public class EntityResumeFilter extends AbstractPathFilter<Entity, Entity, Id> implements Filter<Entity, Entity> {
+
+
+ @Override
+ public Observable<FilterResult<Entity>> call( final Observable<FilterResult<Entity>> filterResultObservable ) {
+
+ //filter only the first id, then map into our path for our next pass
+
+
+ return filterResultObservable.skipWhile( filterResult -> {
+
+ final Optional<Id> startFromCursor = getSeekValue();
+
+ return startFromCursor.isPresent() && startFromCursor.get().equals( filterResult.getValue().getId() );
+ } ).map( filterResult -> {
+
+
+ final Entity entity = filterResult.getValue();
+ final Id entityId = entity.getId();
+
+ return createFilterResult( entity, entityId, filterResult.getPath() );
+ } );
+ }
+
+
+ @Override
+ protected CursorSerializer<Id> getCursorSerializer() {
+ return IdCursorSerializer.INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3a1784f0/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java
index 56e1c1c..0e87141 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java
@@ -28,7 +28,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
-import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
import org.apache.usergrid.corepersistence.pipeline.read.Filter;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
@@ -50,11 +49,10 @@ import rx.Observable;
/**
- * Responsible for verifying candidate result versions, then emitting the Ids of these versions
- * Input is a batch of candidate results, output is a stream of validated Ids
+ * Responsible for verifying candidate result versions, then emitting the Ids of these versions Input is a batch of
+ * candidate results, output is a stream of validated Ids
*/
-public class CandidateIdFilter extends AbstractFilter<Candidate, Id>
- implements Filter<Candidate, Id> {
+public class CandidateIdFilter extends AbstractFilter<Candidate, Id> implements Filter<Candidate, Id> {
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private final EntityIndexFactory entityIndexFactory;
@@ -68,9 +66,8 @@ public class CandidateIdFilter extends AbstractFilter<Candidate, Id>
}
-
@Override
- public Observable<FilterResult<Id>> call( final Observable<FilterResult<Candidate>> filterResultObservable ) {
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Candidate>> filterResultObservable ) {
/**
@@ -87,32 +84,28 @@ public class CandidateIdFilter extends AbstractFilter<Candidate, Id>
final ApplicationEntityIndex applicationIndex =
entityIndexFactory.createApplicationEntityIndex( applicationScope );
- final Observable<FilterResult<Id>> searchIdSetObservable = filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap(
- candidateResults -> {
- //flatten toa list of ids to load
- final Observable<List<Id>> candidateIds =
- Observable.from( candidateResults ).map( candidate -> candidate.getValue().getCandidateResult().getId() )
- .toList();
+ final Observable<FilterResult<Id>> searchIdSetObservable =
+ filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( candidateResults -> {
+ //flatten toa list of ids to load
+ final Observable<List<Id>> candidateIds = Observable.from( candidateResults ).map(
+ candidate -> candidate.getValue().getCandidateResult().getId() ).toList();
- //load the ids
- final Observable<VersionSet> versionSetObservable =
- candidateIds.flatMap( ids -> entityCollectionManager.getLatestVersion( ids ) );
+ //load the ids
+ final Observable<VersionSet> versionSetObservable =
+ candidateIds.flatMap( ids -> entityCollectionManager.getLatestVersion( ids ) );
- //now we have a collection, validate our canidate set is correct.
+ //now we have a collection, validate our canidate set is correct.
- return versionSetObservable.map(
- entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet, candidateResults ) )
- .doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
+ return versionSetObservable.map(
+ entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet,
+ candidateResults ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
entityCollector -> Observable.from( entityCollector.collectResults() ) );
- } );
+ } );
return searchIdSetObservable;
}
-
-
-
/**
* Map a new cp entity to an old entity. May be null if not present
*/
@@ -155,7 +148,6 @@ public class CandidateIdFilter extends AbstractFilter<Candidate, Id>
/**
* Validate each candidate results vs the data loaded from cass
- * @param filterCandidate
*/
private void validate( final FilterResult<Candidate> filterCandidate ) {
@@ -191,11 +183,9 @@ public class CandidateIdFilter extends AbstractFilter<Candidate, Id>
//they're the same add it
- final FilterResult<Id> result = new FilterResult<>( entityId, filterCandidate.getPath() );
+ final FilterResult<Id> result = new FilterResult<>( entityId, filterCandidate.getPath() );
results.add( result );
}
}
-
-
}
[05/12] incubator-usergrid git commit: Updates pipeline and fixes
connectionref querying
Posted by gr...@apache.org.
Updates pipeline and fixes connectionref querying
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/aa9153ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/aa9153ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/aa9153ac
Branch: refs/heads/USERGRID-608
Commit: aa9153ac84a2a7e68dd0c9144ab867c6f75c68a0
Parents: 6d54dff
Author: Todd Nine <tn...@apigee.com>
Authored: Thu May 21 17:10:21 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu May 21 17:10:21 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 21 ++--
.../pipeline/FilterPipeline.java | 107 ------------------
.../corepersistence/pipeline/Pipeline.java | 110 +++++++++++++++++++
.../pipeline/PipelineModule.java | 9 +-
.../pipeline/builder/CandidateBuilder.java | 12 +-
.../pipeline/builder/ConnectionBuilder.java | 37 -------
.../pipeline/builder/ConnectionRefBuilder.java | 6 +-
.../pipeline/builder/EntityBuilder.java | 13 ++-
.../pipeline/builder/IdBuilder.java | 25 ++---
.../pipeline/builder/PipelineBuilder.java | 6 +-
.../results/ConnectionRefQueryExecutor.java | 60 ++++++++++
.../results/EntityQueryExecutor.java | 84 ++++++++++++++
.../results/ObservableQueryExecutor.java | 52 +++------
.../corepersistence/results/QueryExecutor.java | 1 +
14 files changed, 320 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 1c34929..be143ce 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -31,12 +31,12 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
import org.apache.usergrid.corepersistence.pipeline.builder.EntityBuilder;
import org.apache.usergrid.corepersistence.pipeline.builder.IdBuilder;
import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.results.ConnectionRefQueryExecutor;
+import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
import org.apache.usergrid.corepersistence.results.ObservableQueryExecutor;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -640,11 +640,11 @@ public class CpRelationManager implements RelationManager {
}
else {
final String entityType = collection.getType();
- results = pipelineBuilder.searchCollection( collectionName, entityType, query.getQl().get() ).loadEntities();
+ results = pipelineBuilder.searchCollection( collectionName, query.getQl().get() , entityType).loadEntities();
}
- return new ObservableQueryExecutor( results.build() ).next();
+ return new EntityQueryExecutor( results.build() ).next();
}
@@ -936,21 +936,24 @@ public class CpRelationManager implements RelationManager {
if(query.getResultsLevel() == Level.REFS){
- final Observable<ResultsPage<ConnectionRef>> results;
+ final IdBuilder traversedIds;
if(query.isGraphSearch()){
- results = pipelineBuilder.traverseConnection( connection, entityType ).loadConnectionRefs( cpHeadEntity.getId(), connection ).build();
+ traversedIds = pipelineBuilder.traverseConnection( connection, entityType );
}
else
{
- results = pipelineBuilder.searchConnection( connection, query.getQl().get(),entityType) .loadIds().loadConnectionRefs( cpHeadEntity.getId(), connection ).build();
+ traversedIds = pipelineBuilder.searchConnection( connection, query.getQl().get(), entityType ).loadIds();
}
- throw new UnsupportedOperationException( "Implement me" );
+ final Observable<ResultsPage<ConnectionRef>> results = traversedIds.loadConnectionRefs(
+ cpHeadEntity.getId(), connection ).build();
+
+ return new ConnectionRefQueryExecutor( results ).next();
}
@@ -978,7 +981,7 @@ public class CpRelationManager implements RelationManager {
- return new ObservableQueryExecutor( results ).next();
+ return new EntityQueryExecutor( results ).next();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java
deleted file mode 100644
index 089f47d..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline;
-
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-
-import rx.Observable;
-
-
-/**
- * Pipeline for applying our UG domain specific filters.
- *
- * Modeled after an observable, with typing to allow input of specific filters
- *
- * @param InputType the input type in the current pipeline state
- */
-public class FilterPipeline<InputType> {
-
-
- private int idCount = 0;
-
- private final ApplicationScope applicationScope;
-
-
- private final RequestCursor requestCursor;
- private int limit;
-
- //Generics hell, intentionally without a generic, we check at the filter level
- private Observable currentObservable;
-
-
- /**
- * Create our filter pipeline
- */
- public FilterPipeline( final ApplicationScope applicationScope, final Optional<String> cursor, final int limit ) {
-
-
- ValidationUtils.validateApplicationScope( applicationScope );
- Preconditions.checkNotNull( cursor, "cursor optional is required" );
- Preconditions.checkArgument( limit > 0, "limit must be > 0" );
-
-
- this.applicationScope = applicationScope;
-
- //init our cursor to empty
- this.requestCursor = new RequestCursor( cursor );
-
- //set the default limit
- this.limit = limit;
-
- //set our observable to start at the application
- final FilterResult<Id> filter = new FilterResult<>( applicationScope.getApplication(), Optional.absent() );
- this.currentObservable = Observable.just( filter );
- }
-
-
- public <OutputType> FilterPipeline<OutputType> withFilter(
- final PipelineOperation<? super InputType, ? extends OutputType> filter ) {
-
-
-
- final PipelineContext context = new PipelineContext( applicationScope, requestCursor, limit, idCount );
-
- filter.setContext( context );
-
- //done for clarity
- idCount++;
-
- return ( FilterPipeline<OutputType> ) this;
- }
-
-
-
- /**
- * Return the observable of the filter pipeline
- */
- public Observable<InputType> execute() {
- return currentObservable;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java
new file mode 100644
index 0000000..dc95178
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/Pipeline.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+import rx.Observable;
+
+
+/**
+ * Pipeline for applying our UG domain specific filters.
+ *
+ * Modeled after an observable, with typing to allow input of specific filters
+ *
+ * @param InputType the input type in the current pipeline state
+ */
+public class Pipeline<InputType> {
+
+
+ private int idCount = 0;
+
+ private final ApplicationScope applicationScope;
+
+
+ private final RequestCursor requestCursor;
+ private int limit;
+
+ //Generics hell, intentionally without a generic, we check at the filter level
+ private Observable currentObservable;
+
+
+ /**
+ * Create our filter pipeline
+ */
+ public Pipeline( final ApplicationScope applicationScope, final Optional<String> cursor, final int limit ) {
+
+
+ ValidationUtils.validateApplicationScope( applicationScope );
+ Preconditions.checkNotNull( cursor, "cursor optional is required" );
+ Preconditions.checkArgument( limit > 0, "limit must be > 0" );
+
+
+ this.applicationScope = applicationScope;
+
+ //init our cursor to empty
+ this.requestCursor = new RequestCursor( cursor );
+
+ //set the default limit
+ this.limit = limit;
+
+ //set our observable to start at the application
+ final FilterResult<Id> filter = new FilterResult<>( applicationScope.getApplication(), Optional.absent() );
+ this.currentObservable = Observable.just( filter );
+ }
+
+
+ public <OutputType> Pipeline<OutputType> withFilter(
+ final PipelineOperation<? super InputType, ? extends OutputType> filter ) {
+
+
+
+ final PipelineContext context = new PipelineContext( applicationScope, requestCursor, limit, idCount );
+
+ filter.setContext( context );
+
+ //update the observable
+ this.currentObservable = currentObservable.compose( filter );
+
+ //done for clarity
+ idCount++;
+
+ return ( Pipeline<OutputType> ) this;
+ }
+
+
+
+ /**
+ * Return the observable of the filter pipeline
+ */
+ public Observable<InputType> execute() {
+ return currentObservable;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
index 8ec8704..93df066 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline;
+import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
import com.google.inject.AbstractModule;
@@ -33,15 +34,11 @@ public class PipelineModule extends AbstractModule {
@Override
protected void configure() {
- //Use Guice to create the builder since we don't really need to do anything
- //other than DI when creating the filters
-// bind( FilterFactory.class ).to( ReadFilterFactoryImpl.class );
-
-
-// install( new Factory)
//Use Guice to create the builder since we don't really need to do anything
//other than DI when creating the filters
install( new FactoryModuleBuilder().build( FilterFactory.class ) );
+
+ install( new FactoryModuleBuilder().build( PipelineBuilderFactory.class ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java
index 5cb2eab..9354127 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.corepersistence.pipeline.builder;
import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
-import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.Pipeline;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.corepersistence.pipeline.read.search.Candidate;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -31,13 +31,13 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class CandidateBuilder {
- private final FilterPipeline<FilterResult<Candidate>> filterPipeline;
+ private final Pipeline<FilterResult<Candidate>> pipeline;
private final FilterFactory filterFactory;
- public CandidateBuilder( final FilterPipeline<FilterResult<Candidate>> filterPipeline,
+ public CandidateBuilder( final Pipeline<FilterResult<Candidate>> pipeline,
final FilterFactory filterFactory ) {
- this.filterPipeline = filterPipeline;
+ this.pipeline = pipeline;
this.filterFactory = filterFactory;
}
@@ -48,7 +48,7 @@ public class CandidateBuilder {
*/
public IdBuilder loadIds(){
- final FilterPipeline<FilterResult<Id>> newFilter = filterPipeline.withFilter( filterFactory.candidateResultsIdVerifyFilter() );
+ final Pipeline<FilterResult<Id>> newFilter = pipeline.withFilter( filterFactory.candidateResultsIdVerifyFilter() );
return new IdBuilder( newFilter, filterFactory );
}
@@ -60,7 +60,7 @@ public class CandidateBuilder {
*/
public EntityBuilder loadEntities(){
- final FilterPipeline<FilterResult<Entity>> newFilter = filterPipeline.withFilter( filterFactory.candidateEntityFilter() );
+ final Pipeline<FilterResult<Entity>> newFilter = pipeline.withFilter( filterFactory.candidateEntityFilter() );
return new EntityBuilder(newFilter );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java
deleted file mode 100644
index b4ea94e..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.builder;
-
-
-import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
-import org.apache.usergrid.persistence.ConnectionRef;
-
-import rx.Observable;
-
-
-public class ConnectionBuilder {
-
-
-
- public Observable<ResultsPage<ConnectionRef>> build(){
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java
index 6c0ebc8..362f2c6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java
@@ -20,7 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.builder;
-import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.Pipeline;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.corepersistence.pipeline.read.collect.ResultsPageCollector;
@@ -36,9 +36,9 @@ import rx.Observable;
public class ConnectionRefBuilder {
- private final FilterPipeline<FilterResult<ConnectionRef>> connectionRefFilter;
+ private final Pipeline<FilterResult<ConnectionRef>> connectionRefFilter;
- public ConnectionRefBuilder( final FilterPipeline<FilterResult<ConnectionRef>> connectionRefFilter ) {
+ public ConnectionRefBuilder( final Pipeline<FilterResult<ConnectionRef>> connectionRefFilter ) {
this.connectionRefFilter = connectionRefFilter;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java
index 07b4586..b120c56 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java
@@ -20,9 +20,11 @@
package org.apache.usergrid.corepersistence.pipeline.builder;
-import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.Pipeline;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.EntityResumeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.ResultsPageCollector;
import org.apache.usergrid.persistence.model.entity.Entity;
import rx.Observable;
@@ -33,11 +35,11 @@ import rx.Observable;
*/
public class EntityBuilder {
- private final FilterPipeline<FilterResult<Entity>> filterPipeline;
+ private final Pipeline<FilterResult<Entity>> pipeline;
- public EntityBuilder( final FilterPipeline<FilterResult<Entity>> filterPipeline ) {
- this.filterPipeline = filterPipeline;
+ public EntityBuilder( final Pipeline<FilterResult<Entity>> pipeline ) {
+ this.pipeline = pipeline;
}
@@ -46,6 +48,7 @@ public class EntityBuilder {
* @return
*/
public Observable<ResultsPage<Entity>> build(){
- return null;
+ //we must add our resume filter so we drop our previous page first element if it's present
+ return pipeline.withFilter( new EntityResumeFilter() ).withFilter( new ResultsPageCollector<>() ).execute();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
index 12a89ba..4291ea9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
@@ -22,12 +22,11 @@ package org.apache.usergrid.corepersistence.pipeline.builder;
import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
-import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.Pipeline;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefFilter;
import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefResumeFilter;
import org.apache.usergrid.corepersistence.pipeline.read.search.Candidate;
-import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionByTypeFilter;
import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -42,11 +41,11 @@ public class IdBuilder {
private final FilterFactory filterFactory;
- private final FilterPipeline<FilterResult<Id>> filterPipeline;
+ private final Pipeline<FilterResult<Id>> pipeline;
- public IdBuilder( final FilterPipeline<FilterResult<Id>> filterPipeline, final FilterFactory filterFactory ) {
- this.filterPipeline = filterPipeline;
+ public IdBuilder( final Pipeline<FilterResult<Id>> pipeline, final FilterFactory filterFactory ) {
+ this.pipeline = pipeline;
this.filterFactory = filterFactory;
}
@@ -56,8 +55,8 @@ public class IdBuilder {
* @return
*/
public EntityBuilder loadEntities() {
- final FilterPipeline<FilterResult<Entity>> pipeline =
- filterPipeline.withFilter( filterFactory.entityLoadFilter() );
+ final Pipeline<FilterResult<Entity>> pipeline =
+ this.pipeline.withFilter( filterFactory.entityLoadFilter() );
return new EntityBuilder( pipeline );
}
@@ -69,8 +68,8 @@ public class IdBuilder {
* @return
*/
public IdBuilder traverseCollection( final String collectionName ) {
- final FilterPipeline<FilterResult<Id>> newFilter =
- filterPipeline.withFilter( filterFactory.readGraphCollectionFilter( collectionName ) );
+ final Pipeline<FilterResult<Id>> newFilter =
+ pipeline.withFilter( filterFactory.readGraphCollectionFilter( collectionName ) );
return new IdBuilder( newFilter, filterFactory );
}
@@ -93,7 +92,7 @@ public class IdBuilder {
}
- return new IdBuilder( filterPipeline.withFilter(filter ), filterFactory );
+ return new IdBuilder( pipeline.withFilter(filter ), filterFactory );
}
@@ -106,7 +105,7 @@ public class IdBuilder {
*/
public CandidateBuilder searchCollection( final String collectionName, final String ql, final String entityType ) {
- final FilterPipeline<FilterResult<Candidate>> newFilter = filterPipeline.withFilter( filterFactory.searchCollectionFilter(
+ final Pipeline<FilterResult<Candidate>> newFilter = pipeline.withFilter( filterFactory.searchCollectionFilter(
ql, collectionName, entityType ) );
return new CandidateBuilder( newFilter, filterFactory );
@@ -123,7 +122,7 @@ public class IdBuilder {
public CandidateBuilder searchConnection( final String connectionName, final String ql , final Optional<String> entityType) {
- final FilterPipeline<FilterResult<Candidate>> newFilter = filterPipeline.withFilter( filterFactory.searchConnectionFilter(
+ final Pipeline<FilterResult<Candidate>> newFilter = pipeline.withFilter( filterFactory.searchConnectionFilter(
ql, connectionName, entityType ) );
return new CandidateBuilder( newFilter, filterFactory );
@@ -139,7 +138,7 @@ public class IdBuilder {
@Deprecated
public ConnectionRefBuilder loadConnectionRefs(final Id sourceId, final String connectionType){
- final FilterPipeline<FilterResult<ConnectionRef>> connectionRefFilter = filterPipeline.withFilter( new ConnectionRefFilter(sourceId, connectionType ) ).withFilter(
+ final Pipeline<FilterResult<ConnectionRef>> connectionRefFilter = pipeline.withFilter( new ConnectionRefFilter(sourceId, connectionType ) ).withFilter(
new ConnectionRefResumeFilter() );
return new ConnectionRefBuilder(connectionRefFilter);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java
index 488e9c1..f1a44ea 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.corepersistence.pipeline.builder;
import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
-import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.Pipeline;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -91,9 +91,9 @@ public class PipelineBuilder {
*/
@Deprecated
public IdBuilder fromId(final Id entityId){
- FilterPipeline<FilterResult<Id>> filterPipeline = new FilterPipeline( applicationScope, this.cursor,limit ).withFilter( filterFactory.getEntityIdFilter( entityId ) );
+ Pipeline<FilterResult<Id>> pipeline = new Pipeline( applicationScope, this.cursor,limit ).withFilter( filterFactory.getEntityIdFilter( entityId ) );
- return new IdBuilder( filterPipeline, filterFactory );
+ return new IdBuilder( pipeline, filterFactory );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java
new file mode 100644
index 0000000..798c9c7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Processes our results of entities and turns them into
+ */
+@Deprecated//Required for 1.0 compatibility
+public class ConnectionRefQueryExecutor extends ObservableQueryExecutor<ConnectionRef> {
+
+
+ public ConnectionRefQueryExecutor( final Observable<ResultsPage<ConnectionRef>> resultsObservable ) {
+ super( resultsObservable );
+ }
+
+
+ @Override
+ protected Results createResults( final ResultsPage resultsPage ) {
+ final List<ConnectionRef> connectionRefs = resultsPage.getEntityList();
+
+ final Results results = Results.fromConnections(connectionRefs);
+
+ return results;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java
new file mode 100644
index 0000000..bc9001e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Processes our results of entities and turns them into
+ */
+@Deprecated//Required for 1.0 compatibility
+public class EntityQueryExecutor extends ObservableQueryExecutor<Entity> {
+
+
+ public EntityQueryExecutor( final Observable<ResultsPage<Entity>> resultsObservable ) {
+ super( resultsObservable );
+ }
+
+
+ @Override
+ protected Results createResults( final ResultsPage resultsPage ) {
+
+ final List<Entity> entityList = resultsPage.getEntityList();
+ final List<org.apache.usergrid.persistence.Entity> resultsEntities = new ArrayList<>( entityList.size() );
+
+
+ for ( final Entity entity : entityList ) {
+ resultsEntities.add( mapEntity( entity ) );
+ }
+
+ final Results results = Results.fromEntities( resultsEntities );
+
+ return results;
+ }
+
+
+ /**
+ *
+ * @param cpEntity
+ * @return
+ */
+ private org.apache.usergrid.persistence.Entity mapEntity( final Entity cpEntity ) {
+
+
+ final Id entityId = cpEntity.getId();
+
+ org.apache.usergrid.persistence.Entity entity =
+ EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
+
+ Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
+ entity.addProperties( entityMap );
+
+ return entity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
index c779bb7..ff44416 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
@@ -20,9 +20,7 @@
package org.apache.usergrid.corepersistence.results;
-import java.util.ArrayList;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -42,53 +40,42 @@ import rx.Observable;
* Our proxy to allow us to subscribe to observable results, then return them as an iterator. A bridge for 2.0 -> 1.0
* code. This should not be used on any new code, and will eventually be deleted
*/
-public class ObservableQueryExecutor implements QueryExecutor {
+@Deprecated//Required for 1.0 compatibility
+public abstract class ObservableQueryExecutor<T> implements QueryExecutor {
private final Observable<Results> resultsObservable;
public Iterator<Results> iterator;
- public ObservableQueryExecutor( final Observable<ResultsPage<Entity>> resultsObservable) {
- //map to our old results objects, return a default empty if required
- this.resultsObservable = resultsObservable.map( resultsPage -> createResults( resultsPage ) ).defaultIfEmpty( new Results() );
+ public ObservableQueryExecutor( final Observable<ResultsPage<T>> resultsObservable ) {
+ //map to our old results objects, return a default empty if required
+ this.resultsObservable = resultsObservable.map( resultsPage -> createResultsInternal( resultsPage ) )
+ .defaultIfEmpty( new Results() );
}
/**
- *
- * @param cpEntity
+ * Transform the results
+ * @param resultsPage
* @return
*/
- private org.apache.usergrid.persistence.Entity mapEntity( final Entity cpEntity ) {
+ protected abstract Results createResults( final ResultsPage resultsPage );
- final Id entityId = cpEntity.getId();
- org.apache.usergrid.persistence.Entity entity =
- EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
- entity.addProperties( entityMap );
-
- return entity;
- }
-
- private Results createResults( final ResultsPage resultsPage ){
-
- final List<Entity> entityList = resultsPage.getEntityList();
- final List<org.apache.usergrid.persistence.Entity> resultsEntities = new ArrayList<>( entityList.size() );
-
-
- for(final Entity entity: entityList){
- resultsEntities.add( mapEntity( entity ) );
- }
+ /**
+ * Legacy to transform our results page to a new results
+ * @param resultsPage
+ * @return
+ */
+ private Results createResultsInternal( final ResultsPage resultsPage ) {
- final Results results = Results.fromEntities( resultsEntities );
+ final Results results = createResults( resultsPage );
//add the cursor if our limit is the same
- if(resultsPage.hasMoreResults()) {
+ if ( resultsPage.hasMoreResults() ) {
final Optional<String> cursor = resultsPage.getResponseCursor().encodeAsString();
if ( cursor.isPresent() ) {
@@ -96,11 +83,10 @@ public class ObservableQueryExecutor implements QueryExecutor {
}
}
return results;
-
-
}
+
@Override
public Iterator<Results> iterator() {
return this;
@@ -130,6 +116,4 @@ public class ObservableQueryExecutor implements QueryExecutor {
return next;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/aa9153ac/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java
index 3afb77f..6bdf162 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/QueryExecutor.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.persistence.Results;
*
* QueryExecutor.next() should always return a non-null Results object
*/
+@Deprecated//Required for 1.0 compatibility
public interface QueryExecutor extends Iterable<Results>, Iterator<Results> {
[10/12] incubator-usergrid git commit: observable changes
Posted by gr...@apache.org.
observable changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f67961d6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f67961d6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f67961d6
Branch: refs/heads/USERGRID-608
Commit: f67961d6a69a80ab6fb74f3c48bc414588329ba1
Parents: f7b47a8
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue May 26 14:25:09 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue May 26 14:25:09 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 58 ++++++++++----------
1 file changed, 29 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f67961d6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c31288a..bc3c803 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -386,37 +386,37 @@ public class CpRelationManager implements RelationManager {
// create graph edge connection from head entity to member entity
final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
-
- GraphManager gm = managerCache.getGraphManager( applicationScope );
- gm.writeEdge( edge ).toBlocking().last();
- //reverse
-
-
- //perform indexing
-
- if ( logger.isDebugEnabled() ) {
- logger.debug( "Wrote edge {}", edge );
- }
-
-
- //check if we need to reverse our edges
-
-
final String linkedCollection = collection.getLinkedCollection();
- /**
- * Nothing to link
- */
- if ( linkedCollection != null ) {
- String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
- final Edge reverseEdge = createCollectionEdge( memberEntity.getId(), pluralType, cpHeadEntity.getId() );
- gm.writeEdge( reverseEdge ).toBlocking().last();
- }
-
-
- indexService.queueNewEdge( applicationScope, memberEntity, edge );
+ GraphManager gm = managerCache.getGraphManager(applicationScope);
+ gm.writeEdge( edge )
+ .doOnNext( writtenEdge -> {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Wrote edge {}", writtenEdge);
+ }
+ })
+ .filter(writtenEdge -> linkedCollection != null )
+ .flatMap(writtenEdge -> {
+ final String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
+ final Edge reverseEdge = createCollectionEdge( memberEntity.getId(), pluralType, cpHeadEntity.getId() );
+
+ //reverse
+ return gm.writeEdge(reverseEdge).doOnNext(reverseEdgeWritten -> {
+ indexService.queueNewEdge(applicationScope, cpHeadEntity, reverseEdge);
+ });
+ })
+ .doOnCompleted(() -> {
+ indexService.queueNewEdge(applicationScope, memberEntity, edge);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Added entity {}:{} to collection {}", new Object[]{
+ itemRef.getUuid().toString(), itemRef.getType(), collectionName
+ });
+ }
+ })
+ .toBlocking().lastOrDefault(null);
+ //check if we need to reverse our edges
if ( logger.isDebugEnabled() ) {
@@ -622,7 +622,7 @@ public class CpRelationManager implements RelationManager {
final IdBuilder pipelineBuilder =
pipelineBuilderFactory.create( applicationScope ).withCursor( query.getCursor() )
- .withLimit( query.getLimit() ).fromId( cpHeadEntity.getId() );
+ .withLimit( query.getLimit() ).fromId(cpHeadEntity.getId());
final EntityBuilder results;
@@ -904,7 +904,7 @@ public class CpRelationManager implements RelationManager {
query = adjustQuery( query );
- final Optional<String> entityType = Optional.fromNullable( query.getEntityType() ) ;
+ final Optional<String> entityType = Optional.fromNullable(query.getEntityType()) ;
//set startid -- graph | es query filter -- load entities filter (verifies exists) --> results page collector
// -> 1.0 results
[03/12] incubator-usergrid git commit: Refactors operations into
easier build pattern. Pipeline still need some work.
Posted by gr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ResultsPageCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ResultsPageCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ResultsPageCollector.java
index 84654aa..91773c4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ResultsPageCollector.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ResultsPageCollector.java
@@ -23,12 +23,12 @@ package org.apache.usergrid.corepersistence.pipeline.read.collect;
import java.util.ArrayList;
import java.util.List;
+import org.apache.usergrid.corepersistence.pipeline.PipelineContext;
import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
-import org.apache.usergrid.corepersistence.pipeline.read.Collector;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
-import org.apache.usergrid.persistence.model.entity.Entity;
import com.google.common.base.Optional;
@@ -37,21 +37,32 @@ import rx.Observable;
/**
* Takes entities and collects them into results. This mostly exists for 1.0 compatibility. Eventually this will
- * become the only collector in our pipline and be used when rendering results, both on GET, PUT and POST.
+ * become the only collector in our pipeline and be used when rendering results, both on GET, PUT and POST.
+ *
+ *
+ *
+ * @param T the type of element to be collected
*/
-public class ResultsPageCollector extends AbstractCollector<Entity, ResultsPage>
- implements Collector<Entity, ResultsPage> {
+public class ResultsPageCollector<T> extends AbstractFilter<FilterResult<T>, ResultsPage<T>> {
+
+
+ protected PipelineContext pipelineContext;
+
+
+ @Override
+ public void setContext( final PipelineContext pipelineContext ) {
+ this.pipelineContext = pipelineContext;
+ }
+
@Override
- public Observable<ResultsPage> call( final Observable<FilterResult<Entity>> filterResultObservable ) {
+ public Observable<ResultsPage<T>> call( final Observable<FilterResult<T>> filterResultObservable ) {
final int limit = pipelineContext.getLimit();
return filterResultObservable.buffer( limit ).flatMap( buffer -> Observable.from( buffer ).collect(
- () -> new ResultsPageWithCursorCollector( limit ), ( collector, entity ) -> {
- collector.add( entity );
- } ) ).map( resultsPageCollector -> new ResultsPage( resultsPageCollector.results,
+ () -> new ResultsPageWithCursorCollector( limit ), ( collector, element ) -> collector.add( element ) ) ).map( resultsPageCollector -> new ResultsPage( resultsPageCollector.results,
new ResponseCursor( resultsPageCollector.lastPath ), pipelineContext.getLimit() ) );
}
@@ -59,10 +70,10 @@ public class ResultsPageCollector extends AbstractCollector<Entity, ResultsPage>
/**
* A collector that will aggregate our results together
*/
- private static class ResultsPageWithCursorCollector {
+ private class ResultsPageWithCursorCollector {
- private final List<Entity> results;
+ private final List<T> results;
private Optional<EdgePath> lastPath;
@@ -72,7 +83,7 @@ public class ResultsPageCollector extends AbstractCollector<Entity, ResultsPage>
}
- public void add( final FilterResult<Entity> result ) {
+ public void add( final FilterResult<T> result ) {
this.results.add( result.getValue() );
this.lastPath = result.getPath();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractElasticSearchFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractElasticSearchFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractElasticSearchFilter.java
deleted file mode 100644
index f403e21..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractElasticSearchFilter.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.index.CandidateResults;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.codahale.metrics.Timer;
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * Command for reading graph edges
- */
-public abstract class AbstractElasticSearchFilter extends AbstractPathFilter<Id, Candidate, Integer>
- implements Filter<Id, Candidate> {
-
- private static final Logger log = LoggerFactory.getLogger( AbstractElasticSearchFilter.class );
-
- private final EntityIndexFactory entityIndexFactory;
- private final String query;
- private final Timer searchTimer;
-
-
- /**
- * Create a new instance of our command
- */
- public AbstractElasticSearchFilter( final EntityIndexFactory entityIndexFactory,
- final MetricsFactory metricsFactory, final String query ) {
- this.entityIndexFactory = entityIndexFactory;
- this.query = query;
- this.searchTimer = metricsFactory.getTimer( AbstractElasticSearchFilter.class, "query" );
- }
-
-
- @Override
- public Observable<FilterResult<Candidate>> call( final Observable<FilterResult<Id>> observable ) {
-
- //get the graph manager
- final ApplicationEntityIndex applicationEntityIndex =
- entityIndexFactory.createApplicationEntityIndex( pipelineContext.getApplicationScope() );
-
-
- final int limit = pipelineContext.getLimit();
-
-
- final SearchTypes searchTypes = getSearchTypes();
-
-
- //return all ids that are emitted from this edge
- return observable.flatMap( idFilterResult -> {
-
- final SearchEdge searchEdge = getSearchEdge( idFilterResult.getValue() );
-
-
- final Observable<FilterResult<Candidate>> candidates = Observable.create( subscriber -> {
-
- //our offset to our start value. This will be set the first time we emit
- //after we receive new ids, we want to reset this to 0
- //set our our constant state
- final Optional<Integer> startFromCursor = getSeekValue();
-
- final int startOffset = startFromCursor.or( 0 );
-
- int currentOffSet = startOffset;
-
- subscriber.onStart();
-
- //emit while we have values from ES and someone is subscribed
- while ( !subscriber.isUnsubscribed() ) {
-
-
- try {
- final CandidateResults candidateResults =
- applicationEntityIndex.search( searchEdge, searchTypes, query, limit, currentOffSet );
-
-
-
- for( CandidateResult candidateResult: candidateResults){
-
- //our subscriber unsubscribed, break out
- if(subscriber.isUnsubscribed()){
- return;
- }
-
- final Candidate candidate = new Candidate( candidateResult, searchEdge );
-
- final FilterResult<Candidate>
- result = createFilterResult( candidate, currentOffSet, idFilterResult.getPath() );
-
- subscriber.onNext( result );
-
- currentOffSet++;
- }
-
- /**
- * No candidates, we're done
- */
- if (candidateResults.size() < limit) {
- subscriber.onCompleted();
- return;
- }
-
- }
- catch ( Throwable t ) {
-
- log.error( "Unable to search candidates", t );
- subscriber.onError( t );
- }
- }
- } );
-
-
- //add a timer around our observable
- ObservableTimer.time( candidates, searchTimer );
-
- return candidates;
- } );
- }
-
-
- @Override
- protected CursorSerializer<Integer> getCursorSerializer() {
- return ElasticsearchCursorSerializer.INSTANCE;
- }
-
-
- /**
- * Get the search edge from the id
- */
- protected abstract SearchEdge getSearchEdge( final Id id );
-
- /**
- * Get the search types
- */
- protected abstract SearchTypes getSearchTypes();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Candidate.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Candidate.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Candidate.java
deleted file mode 100644
index ab9d5d9..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Candidate.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
-
-
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.index.SearchEdge;
-
-
-/**
- * Create a candidate. This holds the original candidate, as well as the search scope it was found it
- */
-public class Candidate {
-
- private final CandidateResult candidateResult;
- private final SearchEdge searchEdge;
-
-
- /**
- * Create a new Candidate for further processing
- * @param candidateResult The candidate result
- * @param searchEdge The search edge this was searched on
- */
- public Candidate( final CandidateResult candidateResult, final SearchEdge searchEdge ) {
- this.candidateResult = candidateResult;
- this.searchEdge = searchEdge;
- }
-
-
- public CandidateResult getCandidateResult() {
- return candidateResult;
- }
-
-
- public SearchEdge getSearchEdge() {
- return searchEdge;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateEntityFilter.java
deleted file mode 100644
index 4304b37..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateEntityFilter.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-
-import rx.Observable;
-
-
-/**
- * Loads entities from an incoming CandidateResult emissions into entities, then streams them on
- * performs internal buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries. It is up to the
- * collector to determine when to stop streaming entities.
- */
-public class CandidateEntityFilter extends AbstractFilter<Candidate, Entity>
- implements Filter<Candidate, Entity> {
-
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private final EntityIndexFactory entityIndexFactory;
-
-
- @Inject
- public CandidateEntityFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final EntityIndexFactory entityIndexFactory ) {
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.entityIndexFactory = entityIndexFactory;
- }
-
-
- @Override
- public Observable<FilterResult<Entity>> call(
- final Observable<FilterResult<Candidate>> candidateResultsObservable ) {
-
-
- /**
- * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
- * objects
- */
-
- final ApplicationScope applicationScope = pipelineContext.getApplicationScope();
-
- final EntityCollectionManager entityCollectionManager =
- entityCollectionManagerFactory.createCollectionManager( applicationScope );
-
-
- final ApplicationEntityIndex applicationIndex =
- entityIndexFactory.createApplicationEntityIndex( applicationScope );
-
- //buffer them to get a page size we can make 1 network hop
- final Observable<FilterResult<Entity>> searchIdSetObservable = candidateResultsObservable.buffer( pipelineContext.getLimit() )
-
- //load them
- .flatMap( candidateResults -> {
- //flatten toa list of ids to load
- final Observable<List<Id>> candidateIds =
- Observable.from( candidateResults ).map( filterResultCandidate -> filterResultCandidate.getValue().getCandidateResult().getId() ).toList();
-
- //load the ids
- final Observable<EntitySet> entitySetObservable =
- candidateIds.flatMap( ids -> entityCollectionManager.load( ids ) );
-
- //now we have a collection, validate our canidate set is correct.
-
- return entitySetObservable.map(
- entitySet -> new EntityVerifier( applicationIndex.createBatch(), entitySet,
- candidateResults ) ).doOnNext( entityCollector -> entityCollector.merge() )
- .flatMap(
- entityCollector -> Observable.from( entityCollector.getResults() ) );
- } );
-
- //if we filter all our results, we want to continue to try the next page
- return searchIdSetObservable;
- }
-
-
-
-
- /**
- * Our collector to collect entities. Not quite a true collector, but works within our operational flow as this state is mutable and difficult to represent functionally
- */
- private static final class EntityVerifier {
-
- private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
- private List<FilterResult<Entity>> results = new ArrayList<>();
-
- private final EntityIndexBatch batch;
- private final List<FilterResult<Candidate>> candidateResults;
- private final EntitySet entitySet;
-
-
- public EntityVerifier( final EntityIndexBatch batch, final EntitySet entitySet,
- final List<FilterResult<Candidate>> candidateResults ) {
- this.batch = batch;
- this.entitySet = entitySet;
- this.candidateResults = candidateResults;
- this.results = new ArrayList<>( entitySet.size() );
- }
-
-
- /**
- * Merge our candidates and our entity set into results
- */
- public void merge() {
-
- for ( final FilterResult<Candidate> candidateResult : candidateResults ) {
- validate( candidateResult );
- }
-
- batch.execute();
- }
-
-
- public List<FilterResult<Entity>> getResults() {
- return results;
- }
-
-
- public EntityIndexBatch getBatch() {
- return batch;
- }
-
-
- private void validate( final FilterResult<Candidate> filterResult ) {
-
- final Candidate candidate = filterResult.getValue();
- final CandidateResult candidateResult = candidate.getCandidateResult();
- final SearchEdge searchEdge = candidate.getSearchEdge();
- final Id candidateId = candidateResult.getId();
- final UUID candidateVersion = candidateResult.getVersion();
-
-
- final MvccEntity entity = entitySet.getEntity( candidateId );
-
-
- //doesn't exist warn and drop
- if ( entity == null ) {
- logger.warn(
- "Searched and received candidate with entityId {} and version {}, yet was not found in cassandra."
- + " Ignoring since this could be a region sync issue",
- candidateId, candidateVersion );
-
-
- //TODO trigger an audit after a fail count where we explicitly try to repair from other regions
-
- return;
-
- }
-
-
- final UUID entityVersion = entity.getVersion();
- final Id entityId = entity.getId();
-
-
-
-
-
- //entity is newer than ES version, could be an update or the entity is marked as deleted
- if ( UUIDComparator.staticCompare( entityVersion, candidateVersion ) > 0 || !entity.getEntity().isPresent()) {
-
- logger.warn( "Deindexing stale entity on edge {} for entityId {} and version {}",
- new Object[] { searchEdge, entityId, entityVersion } );
- batch.deindex( searchEdge, entityId, candidateVersion );
- return;
- }
-
- //ES is newer than cass, it means we haven't repaired the record in Cass, we don't want to
- //remove the ES record, since the read in cass should cause a read repair, just ignore
- if ( UUIDComparator.staticCompare( candidateVersion, entityVersion ) > 0 ) {
-
- logger.warn(
- "Found a newer version in ES over cassandra for edge {} for entityId {} and version {}. Repair "
- + "should be run", new Object[] { searchEdge, entityId, entityVersion } );
-
- //TODO trigger an audit after a fail count where we explicitly try to repair from other regions
-
- return;
- }
-
- //they're the same add it
-
- final Entity returnEntity = entity.getEntity().get();
-
- final Optional<EdgePath> parent = filterResult.getPath();
-
- final FilterResult<Entity> toReturn = new FilterResult<>( returnEntity, parent );
-
- results.add( toReturn );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java
deleted file mode 100644
index 0e87141..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/CandidateIdFilter.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
-
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.VersionSet;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-import com.google.inject.Inject;
-
-import rx.Observable;
-
-
-/**
- * Responsible for verifying candidate result versions, then emitting the Ids of these versions Input is a batch of
- * candidate results, output is a stream of validated Ids
- */
-public class CandidateIdFilter extends AbstractFilter<Candidate, Id> implements Filter<Candidate, Id> {
-
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private final EntityIndexFactory entityIndexFactory;
-
-
- @Inject
- public CandidateIdFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final EntityIndexFactory entityIndexFactory ) {
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.entityIndexFactory = entityIndexFactory;
- }
-
-
- @Override
- public Observable<FilterResult<Id>> call( final Observable<FilterResult<Candidate>> filterResultObservable ) {
-
-
- /**
- * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
- * objects
- */
-
- final ApplicationScope applicationScope = pipelineContext.getApplicationScope();
-
- final EntityCollectionManager entityCollectionManager =
- entityCollectionManagerFactory.createCollectionManager( applicationScope );
-
-
- final ApplicationEntityIndex applicationIndex =
- entityIndexFactory.createApplicationEntityIndex( applicationScope );
-
- final Observable<FilterResult<Id>> searchIdSetObservable =
- filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( candidateResults -> {
- //flatten toa list of ids to load
- final Observable<List<Id>> candidateIds = Observable.from( candidateResults ).map(
- candidate -> candidate.getValue().getCandidateResult().getId() ).toList();
-
- //load the ids
- final Observable<VersionSet> versionSetObservable =
- candidateIds.flatMap( ids -> entityCollectionManager.getLatestVersion( ids ) );
-
- //now we have a collection, validate our canidate set is correct.
-
- return versionSetObservable.map(
- entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet,
- candidateResults ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
- entityCollector -> Observable.from( entityCollector.collectResults() ) );
- } );
-
- return searchIdSetObservable;
- }
-
-
- /**
- * Map a new cp entity to an old entity. May be null if not present
- */
- private static final class EntityCollector {
-
- private static final Logger logger = LoggerFactory.getLogger( EntityCollector.class );
- private List<FilterResult<Id>> results = new ArrayList<>();
-
- private final EntityIndexBatch batch;
- private final List<FilterResult<Candidate>> candidateResults;
- private final VersionSet versionSet;
-
-
- public EntityCollector( final EntityIndexBatch batch, final VersionSet versionSet,
- final List<FilterResult<Candidate>> candidateResults ) {
- this.batch = batch;
- this.versionSet = versionSet;
- this.candidateResults = candidateResults;
- this.results = new ArrayList<>( versionSet.size() );
- }
-
-
- /**
- * Merge our candidates and our entity set into results
- */
- public void merge() {
-
- for ( final FilterResult<Candidate> candidateResult : candidateResults ) {
- validate( candidateResult );
- }
-
- batch.execute();
- }
-
-
- public List<FilterResult<Id>> collectResults() {
- return results;
- }
-
-
- /**
- * Validate each candidate results vs the data loaded from cass
- */
- private void validate( final FilterResult<Candidate> filterCandidate ) {
-
- final CandidateResult candidateResult = filterCandidate.getValue().getCandidateResult();
-
- final SearchEdge searchEdge = filterCandidate.getValue().getSearchEdge();
-
- final MvccLogEntry logEntry = versionSet.getMaxVersion( candidateResult.getId() );
-
- final UUID candidateVersion = candidateResult.getVersion();
-
- final UUID entityVersion = logEntry.getVersion();
-
- final Id entityId = logEntry.getEntityId();
-
- //entity is newer than ES version
- if ( UUIDComparator.staticCompare( entityVersion, candidateVersion ) > 0 ) {
-
- logger.warn( "Deindexing stale entity on edge {} for entityId {} and version {}",
- new Object[] { searchEdge, entityId, entityVersion } );
- batch.deindex( searchEdge, entityId, entityVersion );
- return;
- }
-
- //ES is newer than cass, it means we haven't repaired the record in Cass, we don't want to
- //remove the ES record, since the read in cass should cause a read repair, just ignore
- if ( UUIDComparator.staticCompare( candidateVersion, entityVersion ) > 0 ) {
-
- logger.warn(
- "Found a newer version in ES over cassandra for edge {} for entityId {} and version {}. Repair "
- + "should be run", new Object[] { searchEdge, entityId, entityVersion } );
- }
-
- //they're the same add it
-
- final FilterResult<Id> result = new FilterResult<>( entityId, filterCandidate.getPath() );
-
- results.add( result );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchCollectionFilter.java
deleted file mode 100644
index 702b2d9..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchCollectionFilter.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
-
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionSearchEdge;
-
-
-public class ElasticSearchCollectionFilter extends AbstractElasticSearchFilter {
-
- private final String collectionName;
- private final String entityType;
-
- /**
- * Create a new instance of our command
- *
- * @param entityIndexFactory The entity index factory used to search
- * @param metricsFactory The metrics factory for metrics
- * @param collectionName The name of the collection
- * @param entityType The entity type
- */
- @Inject
- public ElasticSearchCollectionFilter( final EntityIndexFactory entityIndexFactory,
- final MetricsFactory metricsFactory, @Assisted( "query" ) final String query,
- @Assisted( "collectionName" ) final String collectionName,
- @Assisted( "entityType" ) final String entityType ) {
- super( entityIndexFactory, metricsFactory, query );
- this.collectionName = collectionName;
- this.entityType = entityType;
- }
-
-
-
- @Override
- protected SearchTypes getSearchTypes() {
- final SearchTypes types = SearchTypes.fromTypes( entityType );
-
- return types;
- }
-
-
- @Override
- protected SearchEdge getSearchEdge( final Id incomingId ) {
- final SearchEdge searchEdge = createCollectionSearchEdge( incomingId, collectionName );
-
- return searchEdge;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchConnectionFilter.java
deleted file mode 100644
index cc40530..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticSearchConnectionFilter.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
-
-
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
-import org.apache.usergrid.persistence.index.EntityIndexFactory;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchEdge;
-
-
-public class ElasticSearchConnectionFilter extends AbstractElasticSearchFilter {
-
-
- private final String connectionName;
- private final Optional<String> connectedEntityType;
-
-
- /**
- * Create a new instance of our command
- */
- @Inject
- public ElasticSearchConnectionFilter( final EntityIndexFactory entityIndexFactory,
- final MetricsFactory metricsFactory, @Assisted( "query" ) final String query,
- @Assisted( "connectionName" ) final String connectionName,
- @Assisted( "connectedEntityType" )
- final Optional<String> connectedEntityType ) {
- super( entityIndexFactory, metricsFactory, query );
-
- this.connectionName = connectionName;
- this.connectedEntityType = connectedEntityType;
- }
-
-
- @Override
- protected SearchTypes getSearchTypes() {
- final SearchTypes searchTypes = SearchTypes.fromNullableTypes( connectedEntityType.orNull() );
-
- return searchTypes;
- }
-
-
- @Override
- protected SearchEdge getSearchEdge( final Id id ) {
- final SearchEdge searchEdge = createConnectionSearchEdge( id, connectionName );
-
- return searchEdge;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticsearchCursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticsearchCursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticsearchCursorSerializer.java
deleted file mode 100644
index a4e7746..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/ElasticsearchCursorSerializer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
-
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.AbstractCursorSerializer;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
-
-
-/**
- * ElasticSearch cursor serializer
- */
-public class ElasticsearchCursorSerializer extends AbstractCursorSerializer<Integer> {
-
-
- public static final ElasticsearchCursorSerializer INSTANCE = new ElasticsearchCursorSerializer();
-
- @Override
- protected Class<Integer> getType() {
- return Integer.class;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Elasticsearchdiagram.jpg
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Elasticsearchdiagram.jpg b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Elasticsearchdiagram.jpg
deleted file mode 100644
index 08970e3..0000000
Binary files a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/Elasticsearchdiagram.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java
deleted file mode 100644
index 42b352b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.SearchByEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import rx.Observable;
-
-
-/**
- * Filter should take and Id and a graph edge, and ensure the connection between the two exists
- */
-public abstract class AbstractReadGraphEdgeByIdFilter extends AbstractFilter<Id, Id> implements
- Filter<Id, Id> {
-
- private final GraphManagerFactory graphManagerFactory;
- private final Id targetId;
-
-
- @Inject
- public AbstractReadGraphEdgeByIdFilter( final GraphManagerFactory graphManagerFactory, @Assisted final Id
- targetId ) {
- this.graphManagerFactory = graphManagerFactory;
- this.targetId = targetId;
- }
-
-
- @Override
- public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterValueObservable ) {
-
- final GraphManager gm = graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() );
-
- return filterValueObservable.flatMap( filterValue -> {
- final String edgeTypeName = getEdgeName();
- final Id id = filterValue.getValue();
-
- //create our search
- final SearchByEdge searchByEdge =
- new SimpleSearchByEdge( id, edgeTypeName, targetId, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- Optional.absent() );
-
- //load the versions of the edge, take the first since that's all we need to validate existence, then emit the target node
- return gm.loadEdgeVersions( searchByEdge ).take( 1 ).map( edge -> edge.getTargetNode() ).map( targetId -> new FilterResult<>(targetId, filterValue.getPath()));
- } );
- }
-
-
- /**
- * Get the name of the edge to be used in the seek
- */
- protected abstract String getEdgeName();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
deleted file mode 100644
index 303bc5b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * Command for reading graph edges
- */
-public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id, Edge> implements Filter<Id, Id> {
-
- private final GraphManagerFactory graphManagerFactory;
-
-
- /**
- * Create a new instance of our command
- */
- public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory ) {
- this.graphManagerFactory = graphManagerFactory;
- }
-
-
- @Override
- public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> previousIds ) {
-
-
- //get the graph manager
- final GraphManager graphManager =
- graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() );
-
-
- final String edgeName = getEdgeTypeName();
- final EdgeState edgeCursorState = new EdgeState();
-
-
- //return all ids that are emitted from this edge
- return previousIds.flatMap( previousFilterValue -> {
-
- //set our our constant state
- final Optional<Edge> startFromCursor = getSeekValue();
- final Id id = previousFilterValue.getValue();
-
-
- final SimpleSearchByEdgeType search =
- new SimpleSearchByEdgeType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- startFromCursor );
-
- /**
- * TODO, pass a message with pointers to our cursor values to be generated later
- */
- return graphManager.loadEdgesFromSource( search )
- //set the edge state for cursors
- .doOnNext( edge -> edgeCursorState.update( edge ) )
-
- //map our id from the target edge and set our cursor every edge we traverse
- .map( edge -> createFilterResult( edge.getTargetNode(), edgeCursorState.getCursorEdge(),
- previousFilterValue.getPath() ) );
- } );
- }
-
-
- @Override
- protected FilterResult<Id> createFilterResult( final Id emit, final Edge cursorValue,
- final Optional<EdgePath> parent ) {
-
- //if it's our first pass, there's no cursor to generate
- if(cursorValue == null){
- return new FilterResult<>( emit, parent );
- }
-
- return super.createFilterResult( emit, cursorValue, parent );
- }
-
-
- @Override
- protected CursorSerializer<Edge> getCursorSerializer() {
- return EdgeCursorSerializer.INSTANCE;
- }
-
-
- /**
- * Get the edge type name we should use when traversing
- */
- protected abstract String getEdgeTypeName();
-
-
- /**
- * Wrapper class. Because edges seek > the last returned, we need to keep our n-1 value. This will be our cursor We
- * always try to seek to the same position as we ended. Since we don't deal with a persistent read result, if we
- * seek to a value = to our last, we may skip data.
- */
- private final class EdgeState {
-
- private Edge cursorEdge = null;
- private Edge currentEdge = null;
-
-
- /**
- * Update the pointers
- */
- private void update( final Edge newEdge ) {
- cursorEdge = currentEdge;
- currentEdge = newEdge;
- }
-
-
- /**
- * Get the edge to use in cursors for resume
- */
- private Edge getCursorEdge() {
- return cursorEdge;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EdgeCursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EdgeCursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EdgeCursorSerializer.java
deleted file mode 100644
index 769a67e..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EdgeCursorSerializer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.AbstractCursorSerializer;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
-
-
-/**
- * Edge cursor serializer
- */
-public class EdgeCursorSerializer extends AbstractCursorSerializer<Edge> {
-
-
- public static final EdgeCursorSerializer INSTANCE = new EdgeCursorSerializer();
-
- @Override
- protected Class<SimpleEdge> getType() {
- return SimpleEdge.class;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityIdFilter.java
deleted file mode 100644
index 5a0e026..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityIdFilter.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import rx.Observable;
-
-
-/**
- * This command is a stopgap to make migrating 1.0 code easier. Once full traversal has been implemented, this should
- * be removed
- */
-public class EntityIdFilter extends AbstractFilter<Id, Id> implements Filter<Id, Id> {
-
- private final Id entityId;
-
-
- @Inject
- public EntityIdFilter( @Assisted final Id entityId ) {this.entityId = entityId;}
-
-
-
- @Override
- public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterValueObservable ) {
- //ignore what our input was, and simply emit the id specified
- return filterValueObservable.map( idFilterResult -> new FilterResult( entityId, idFilterResult.getPath() ));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityLoadFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityLoadFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityLoadFilter.java
deleted file mode 100644
index d598a2e..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/EntityLoadFilter.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-
-import rx.Observable;
-
-
-/**
- * Loads entities from a set of Ids.
- *
- * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
- */
-public class EntityLoadFilter extends AbstractFilter<Id, Entity> implements Filter<Id, Entity> {
-
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
-
-
- @Inject
- public EntityLoadFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- }
-
-
- @Override
- public Observable<FilterResult<Entity>> call( final Observable<FilterResult<Id>> filterResultObservable ) {
-
-
- final EntityCollectionManager entityCollectionManager =
- entityCollectionManagerFactory.createCollectionManager( pipelineContext.getApplicationScope() );
-
- //it's more efficient to make 1 network hop to get everything, then drop our results if required
- final Observable<FilterResult<Entity>> entityObservable =
- filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( bufferedIds -> {
-
- final Observable<EntitySet> entitySetObservable =
- Observable.from( bufferedIds ).map( filterResultId -> filterResultId.getValue() ).toList()
- .flatMap( ids -> entityCollectionManager.load( ids ) );
-
-
- //now we have a collection, validate our canidate set is correct.
-
- return entitySetObservable.map( entitySet -> new EntityVerifier( entitySet, bufferedIds ) )
- .doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
- entityCollector -> Observable.from( entityCollector.getResults() ) );
- } );
-
- return entityObservable;
- }
-
-
- /**
- * Our collector to collect entities. Not quite a true collector, but works within our operational flow as this
- * state is mutable and difficult to represent functionally
- */
- private static final class EntityVerifier {
-
- private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
- private List<FilterResult<Entity>> results = new ArrayList<>();
-
- private final List<FilterResult<Id>> candidateResults;
- private final EntitySet entitySet;
-
-
- public EntityVerifier( final EntitySet entitySet, final List<FilterResult<Id>> candidateResults ) {
- this.entitySet = entitySet;
- this.candidateResults = candidateResults;
- this.results = new ArrayList<>( entitySet.size() );
- }
-
-
- /**
- * Merge our candidates and our entity set into results
- */
- public void merge() {
-
- for ( final FilterResult<Id> candidateResult : candidateResults ) {
- validate( candidateResult );
- }
- }
-
-
- public List<FilterResult<Entity>> getResults() {
- return results;
- }
-
-
- private void validate( final FilterResult<Id> filterResult ) {
-
- final Id candidateId = filterResult.getValue();
-
-
- final MvccEntity entity = entitySet.getEntity( candidateId );
-
-
- //doesn't exist warn and drop
- if ( entity == null || !entity.getEntity().isPresent() ) {
- logger.warn( "Read graph edge and received candidate with entityId {}, yet was not found in cassandra."
- + " Ignoring since this could be a region sync issue", candidateId );
-
-
- //TODO trigger an audit after a fail count where we explicitly try to repair from other regions
-
- return;
- }
-
- //it exists, add it
-
- final Entity returnEntity = entity.getEntity().get();
-
- final Optional<EdgePath> parent = filterResult.getPath();
-
- final FilterResult<Entity> toReturn = new FilterResult<>( returnEntity, parent );
-
- results.add( toReturn );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/GraphDiagram.jpg
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/GraphDiagram.jpg b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/GraphDiagram.jpg
deleted file mode 100644
index c0308bd..0000000
Binary files a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/GraphDiagram.jpg and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
deleted file mode 100644
index da6ad29..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-
-/**
- * Read an edge in the graph to verify it's existence by id
- */
-public class ReadGraphCollectionByIdFilter extends AbstractReadGraphEdgeByIdFilter{
-
- private final String collectionName;
-
- @Inject
- public ReadGraphCollectionByIdFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName, @Assisted final Id targetId ) {
- super( graphManagerFactory, targetId );
- this.collectionName = collectionName;
- }
-
-
- @Override
- protected String getEdgeName() {
- return CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
deleted file mode 100644
index 91ae7c3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromCollectionName;
-
-
-/**
- * Command for reading graph edges on a collection
- */
-public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
-
- private final String collectionName;
-
-
- /**
- * Create a new instance of our command
- */
- @Inject
- public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName ) {
- super( graphManagerFactory );
- this.collectionName = collectionName;
- }
-
-
- @Override
- protected String getEdgeTypeName() {
- return getEdgeTypeFromCollectionName( collectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
deleted file mode 100644
index 4756d33..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-
-/**
- * Read an edge in the graph to verify it's existence by id
- */
-public class ReadGraphConnectionByIdFilter extends AbstractReadGraphEdgeByIdFilter{
-
- private final String connectionName;
-
- @Inject
- public ReadGraphConnectionByIdFilter( final GraphManagerFactory graphManagerFactory,
- @Assisted final String connectionName, @Assisted final Id targetId ) {
- super( graphManagerFactory, targetId );
- this.connectionName = connectionName;
- }
-
-
- @Override
- protected String getEdgeName() {
- return CpNamingUtils.getEdgeTypeFromConnectionType( connectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
deleted file mode 100644
index 7371579..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
-import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import rx.Observable;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromConnectionType;
-
-
-/**
- * Command for reading graph edges on a connection
- */
-public class ReadGraphConnectionByTypeFilter extends AbstractPathFilter<Id, Id, Edge> implements Filter<Id, Id> {
-
- private final GraphManagerFactory graphManagerFactory;
- private final String connectionName;
- private final String entityType;
-
-
- /**
- * Create a new instance of our command
- */
- @Inject
- public ReadGraphConnectionByTypeFilter( final GraphManagerFactory graphManagerFactory,
- @Assisted("connectionName") final String connectionName, @Assisted("entityType") final String entityType ) {
- this.graphManagerFactory = graphManagerFactory;
- this.connectionName = connectionName;
- this.entityType = entityType;
- }
-
-
-
- @Override
- public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterResultObservable ) {
-
- //get the graph manager
- final GraphManager graphManager = graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() );
-
-
-
- final String edgeName = getEdgeTypeFromConnectionType( connectionName );
-
-
- //return all ids that are emitted from this edge
- return filterResultObservable.flatMap( idFilterResult -> {
-
- //set our our constant state
- final Optional<Edge> startFromCursor = getSeekValue();
- final Id id = idFilterResult.getValue();
-
- final SimpleSearchByIdType search =
- new SimpleSearchByIdType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- entityType, startFromCursor );
-
- return graphManager.loadEdgesFromSourceByType( search ).map(
- edge -> createFilterResult( edge.getTargetNode(), edge, idFilterResult.getPath() ));
- } );
- }
-
-
- @Override
- protected CursorSerializer<Edge> getCursorSerializer() {
- return EdgeCursorSerializer.INSTANCE;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
deleted file mode 100644
index 0d4971b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.graph;
-
-
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromConnectionType;
-
-
-/**
- * Command for reading graph edges on a connection
- */
-public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
-
- private final String connectionName;
-
-
- /**
- * Create a new instance of our command
- */
- @Inject
- public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String connectionName ) {
- super( graphManagerFactory );
- this.connectionName = connectionName;
- }
-
-
- @Override
- protected String getEdgeTypeName() {
- return getEdgeTypeFromConnectionType( connectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
new file mode 100644
index 0000000..eaf74c1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/AbstractElasticSearchFilter.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.search;
+
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.core.metrics.ObservableTimer;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.index.CandidateResults;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.codahale.metrics.Timer;
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * Command for reading graph edges
+ */
+public abstract class AbstractElasticSearchFilter extends AbstractPathFilter<Id, Candidate, Integer> {
+
+ private static final Logger log = LoggerFactory.getLogger( AbstractElasticSearchFilter.class );
+
+ private final EntityIndexFactory entityIndexFactory;
+ private final String query;
+ private final Timer searchTimer;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ public AbstractElasticSearchFilter( final EntityIndexFactory entityIndexFactory,
+ final MetricsFactory metricsFactory, final String query ) {
+ this.entityIndexFactory = entityIndexFactory;
+ this.query = query;
+ this.searchTimer = metricsFactory.getTimer( AbstractElasticSearchFilter.class, "query" );
+ }
+
+
+ @Override
+ public Observable<FilterResult<Candidate>> call( final Observable<FilterResult<Id>> observable ) {
+
+ //get the graph manager
+ final ApplicationEntityIndex applicationEntityIndex =
+ entityIndexFactory.createApplicationEntityIndex( pipelineContext.getApplicationScope() );
+
+
+ final int limit = pipelineContext.getLimit();
+
+
+ final SearchTypes searchTypes = getSearchTypes();
+
+
+ //return all ids that are emitted from this edge
+ return observable.flatMap( idFilterResult -> {
+
+ final SearchEdge searchEdge = getSearchEdge( idFilterResult.getValue() );
+
+
+ final Observable<FilterResult<Candidate>> candidates = Observable.create( subscriber -> {
+
+ //our offset to our start value. This will be set the first time we emit
+ //after we receive new ids, we want to reset this to 0
+ //set our our constant state
+ final Optional<Integer> startFromCursor = getSeekValue();
+
+ final int startOffset = startFromCursor.or( 0 );
+
+ int currentOffSet = startOffset;
+
+ subscriber.onStart();
+
+ //emit while we have values from ES and someone is subscribed
+ while ( !subscriber.isUnsubscribed() ) {
+
+
+ try {
+ final CandidateResults candidateResults =
+ applicationEntityIndex.search( searchEdge, searchTypes, query, limit, currentOffSet );
+
+
+
+ for( CandidateResult candidateResult: candidateResults){
+
+ //our subscriber unsubscribed, break out
+ if(subscriber.isUnsubscribed()){
+ return;
+ }
+
+ final Candidate candidate = new Candidate( candidateResult, searchEdge );
+
+ final FilterResult<Candidate>
+ result = createFilterResult( candidate, currentOffSet, idFilterResult.getPath() );
+
+ subscriber.onNext( result );
+
+ currentOffSet++;
+ }
+
+ /**
+ * No candidates, we're done
+ */
+ if (candidateResults.size() < limit) {
+ subscriber.onCompleted();
+ return;
+ }
+
+ }
+ catch ( Throwable t ) {
+
+ log.error( "Unable to search candidates", t );
+ subscriber.onError( t );
+ }
+ }
+ } );
+
+
+ //add a timer around our observable
+ ObservableTimer.time( candidates, searchTimer );
+
+ return candidates;
+ } );
+ }
+
+
+ @Override
+ protected CursorSerializer<Integer> getCursorSerializer() {
+ return ElasticsearchCursorSerializer.INSTANCE;
+ }
+
+
+ /**
+ * Get the search edge from the id
+ */
+ protected abstract SearchEdge getSearchEdge( final Id id );
+
+ /**
+ * Get the search types
+ */
+ protected abstract SearchTypes getSearchTypes();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Candidate.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Candidate.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Candidate.java
new file mode 100644
index 0000000..7ada4ba
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Candidate.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.search;
+
+
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Create a candidate. This holds the original candidate, as well as the search scope it was found it
+ */
+public class Candidate {
+
+ private final CandidateResult candidateResult;
+ private final SearchEdge searchEdge;
+
+
+ /**
+ * Create a new Candidate for further processing
+ * @param candidateResult The candidate result
+ * @param searchEdge The search edge this was searched on
+ */
+ public Candidate( final CandidateResult candidateResult, final SearchEdge searchEdge ) {
+ this.candidateResult = candidateResult;
+ this.searchEdge = searchEdge;
+ }
+
+
+ public CandidateResult getCandidateResult() {
+ return candidateResult;
+ }
+
+
+ public SearchEdge getSearchEdge() {
+ return searchEdge;
+ }
+}
[11/12] incubator-usergrid git commit: Merge branch 'pr/254' into
two-dot-o-dev
Posted by gr...@apache.org.
Merge branch 'pr/254' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1396ebe1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1396ebe1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1396ebe1
Branch: refs/heads/USERGRID-608
Commit: 1396ebe1bc2db937744bf077101dc977c1a917a9
Parents: 5bb9aff f67961d
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue May 26 15:17:26 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue May 26 15:17:26 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 28 +-
.../corepersistence/CpEntityManagerFactory.java | 7 +-
.../corepersistence/CpRelationManager.java | 219 ++++++++------
.../corepersistence/index/IndexServiceImpl.java | 46 +--
.../corepersistence/pipeline/Pipeline.java | 93 +++---
.../pipeline/PipelineBuilderFactory.java | 39 ---
.../pipeline/PipelineModule.java | 20 +-
.../pipeline/PipelineOperation.java | 2 +-
.../pipeline/PipelineOperations.java | 30 ++
.../pipeline/builder/CandidateBuilder.java | 67 +++++
.../pipeline/builder/ConnectionRefBuilder.java | 53 ++++
.../pipeline/builder/EntityBuilder.java | 54 ++++
.../pipeline/builder/IdBuilder.java | 151 ++++++++++
.../pipeline/builder/PipelineBuilder.java | 100 +++++++
.../builder/PipelineBuilderFactory.java | 35 +++
.../pipeline/read/AbstractFilter.java | 2 +-
.../pipeline/read/AbstractPathFilter.java | 2 +-
.../pipeline/read/Collector.java | 38 ---
.../pipeline/read/CollectorFactory.java | 38 ---
.../corepersistence/pipeline/read/Filter.java | 31 --
.../pipeline/read/FilterFactory.java | 74 +++--
.../pipeline/read/ReadFilterFactoryImpl.java | 136 ---------
.../pipeline/read/ReadPipelineBuilder.java | 104 -------
.../pipeline/read/ReadPipelineBuilderImpl.java | 296 -------------------
.../pipeline/read/ResultsPage.java | 10 +-
.../read/collect/AbstractCollector.java | 46 ---
.../read/collect/ConnectionRefFilter.java | 68 +++++
.../read/collect/ConnectionRefResumeFilter.java | 86 ++++++
.../pipeline/read/collect/EntityFilter.java | 68 -----
.../read/collect/EntityResumeFilter.java | 67 +++++
.../pipeline/read/collect/IdResumeFilter.java | 61 ++++
.../read/collect/ResultsPageCollector.java | 35 ++-
.../AbstractElasticSearchFilter.java | 171 -----------
.../pipeline/read/elasticsearch/Candidate.java | 55 ----
.../elasticsearch/CandidateEntityFilter.java | 234 ---------------
.../read/elasticsearch/CandidateIdFilter.java | 201 -------------
.../ElasticSearchCollectionFilter.java | 77 -----
.../ElasticSearchConnectionFilter.java | 73 -----
.../ElasticsearchCursorSerializer.java | 42 ---
.../read/elasticsearch/Elasticsearchdiagram.jpg | Bin 316655 -> 0 bytes
.../graph/AbstractReadGraphEdgeByIdFilter.java | 82 -----
.../read/graph/AbstractReadGraphFilter.java | 147 ---------
.../read/graph/EdgeCursorSerializer.java | 42 ---
.../pipeline/read/graph/EntityIdFilter.java | 54 ----
.../pipeline/read/graph/EntityLoadFilter.java | 155 ----------
.../pipeline/read/graph/GraphDiagram.jpg | Bin 347711 -> 0 bytes
.../graph/ReadGraphCollectionByIdFilter.java | 49 ---
.../read/graph/ReadGraphCollectionFilter.java | 53 ----
.../graph/ReadGraphConnectionByIdFilter.java | 50 ----
.../graph/ReadGraphConnectionByTypeFilter.java | 100 -------
.../read/graph/ReadGraphConnectionFilter.java | 53 ----
.../search/AbstractElasticSearchFilter.java | 169 +++++++++++
.../pipeline/read/search/Candidate.java | 55 ++++
.../read/search/CandidateEntityFilter.java | 232 +++++++++++++++
.../pipeline/read/search/CandidateIdFilter.java | 190 ++++++++++++
.../search/ElasticsearchCursorSerializer.java | 40 +++
.../read/search/Elasticsearchdiagram.jpg | Bin 0 -> 316655 bytes
.../read/search/SearchCollectionFilter.java | 77 +++++
.../read/search/SearchConnectionFilter.java | 72 +++++
.../AbstractReadGraphEdgeByIdFilter.java | 82 +++++
.../read/traverse/AbstractReadGraphFilter.java | 146 +++++++++
.../read/traverse/EdgeCursorSerializer.java | 42 +++
.../pipeline/read/traverse/EntityIdFilter.java | 53 ++++
.../read/traverse/EntityLoadVerifyFilter.java | 154 ++++++++++
.../pipeline/read/traverse/GraphDiagram.jpg | Bin 0 -> 347711 bytes
.../traverse/ReadGraphCollectionByIdFilter.java | 49 +++
.../traverse/ReadGraphCollectionFilter.java | 53 ++++
.../traverse/ReadGraphConnectionByIdFilter.java | 50 ++++
.../ReadGraphConnectionByTypeFilter.java | 99 +++++++
.../traverse/ReadGraphConnectionFilter.java | 53 ++++
.../results/ConnectionRefQueryExecutor.java | 60 ++++
.../results/EntityQueryExecutor.java | 84 ++++++
.../results/ObservableQueryExecutor.java | 52 ++--
.../corepersistence/results/QueryExecutor.java | 1 +
.../pipeline/cursor/CursorTest.java | 4 +-
75 files changed, 2830 insertions(+), 2731 deletions(-)
----------------------------------------------------------------------
[04/12] incubator-usergrid git commit: Refactors operations into
easier build pattern. Pipeline still need some work.
Posted by gr...@apache.org.
Refactors operations into easier build pattern. Pipeline still need some work.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6d54dffc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6d54dffc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6d54dffc
Branch: refs/heads/USERGRID-608
Commit: 6d54dffc4e9178b85349ec591275c9005ad121ed
Parents: 3a1784f
Author: Todd Nine <tn...@apigee.com>
Authored: Wed May 20 15:00:38 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed May 20 19:24:50 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 16 +-
.../corepersistence/CpEntityManagerFactory.java | 12 +-
.../corepersistence/CpRelationManager.java | 93 +++++---
.../pipeline/FilterPipeline.java | 107 +++++++++
.../pipeline/PipelineModule.java | 2 -
.../pipeline/PipelineOperation.java | 2 +-
.../pipeline/builder/CandidateBuilder.java | 67 ++++++
.../pipeline/builder/ConnectionBuilder.java | 37 +++
.../pipeline/builder/ConnectionRefBuilder.java | 53 +++++
.../pipeline/builder/EntityBuilder.java | 51 ++++
.../pipeline/builder/IdBuilder.java | 147 ++++++++++++
.../pipeline/builder/PipelineBuilder.java | 100 ++++++++
.../builder/PipelineBuilderFactory.java | 35 +++
.../pipeline/read/AbstractFilter.java | 2 +-
.../pipeline/read/AbstractPathFilter.java | 2 +-
.../pipeline/read/Collector.java | 38 ---
.../pipeline/read/CollectorFactory.java | 38 ---
.../corepersistence/pipeline/read/Filter.java | 38 ---
.../pipeline/read/FilterFactory.java | 69 ++++--
.../pipeline/read/FilterPipeline.java | 132 -----------
.../pipeline/read/ReadFilterFactoryImpl.java | 136 -----------
.../pipeline/read/ResultsPage.java | 10 +-
.../read/collect/AbstractCollector.java | 46 ----
.../read/collect/ConnectionRefFilter.java | 68 ++++++
.../read/collect/ConnectionRefResumeFilter.java | 86 +++++++
.../read/collect/EntityResumeFilter.java | 3 +-
.../read/collect/ResultsPageCollector.java | 35 ++-
.../AbstractElasticSearchFilter.java | 171 --------------
.../pipeline/read/elasticsearch/Candidate.java | 55 -----
.../elasticsearch/CandidateEntityFilter.java | 234 -------------------
.../read/elasticsearch/CandidateIdFilter.java | 191 ---------------
.../ElasticSearchCollectionFilter.java | 77 ------
.../ElasticSearchConnectionFilter.java | 73 ------
.../ElasticsearchCursorSerializer.java | 42 ----
.../read/elasticsearch/Elasticsearchdiagram.jpg | Bin 316655 -> 0 bytes
.../graph/AbstractReadGraphEdgeByIdFilter.java | 82 -------
.../read/graph/AbstractReadGraphFilter.java | 147 ------------
.../read/graph/EdgeCursorSerializer.java | 42 ----
.../pipeline/read/graph/EntityIdFilter.java | 54 -----
.../pipeline/read/graph/EntityLoadFilter.java | 155 ------------
.../pipeline/read/graph/GraphDiagram.jpg | Bin 347711 -> 0 bytes
.../graph/ReadGraphCollectionByIdFilter.java | 49 ----
.../read/graph/ReadGraphCollectionFilter.java | 53 -----
.../graph/ReadGraphConnectionByIdFilter.java | 50 ----
.../graph/ReadGraphConnectionByTypeFilter.java | 100 --------
.../read/graph/ReadGraphConnectionFilter.java | 53 -----
.../search/AbstractElasticSearchFilter.java | 169 ++++++++++++++
.../pipeline/read/search/Candidate.java | 55 +++++
.../read/search/CandidateEntityFilter.java | 232 ++++++++++++++++++
.../pipeline/read/search/CandidateIdFilter.java | 190 +++++++++++++++
.../search/ElasticsearchCursorSerializer.java | 40 ++++
.../read/search/Elasticsearchdiagram.jpg | Bin 0 -> 316655 bytes
.../read/search/SearchCollectionFilter.java | 77 ++++++
.../read/search/SearchConnectionFilter.java | 72 ++++++
.../AbstractReadGraphEdgeByIdFilter.java | 82 +++++++
.../read/traverse/AbstractReadGraphFilter.java | 146 ++++++++++++
.../read/traverse/EdgeCursorSerializer.java | 42 ++++
.../pipeline/read/traverse/EntityIdFilter.java | 53 +++++
.../read/traverse/EntityLoadVerifyFilter.java | 154 ++++++++++++
.../pipeline/read/traverse/GraphDiagram.jpg | Bin 0 -> 347711 bytes
.../traverse/ReadGraphCollectionByIdFilter.java | 49 ++++
.../traverse/ReadGraphCollectionFilter.java | 53 +++++
.../traverse/ReadGraphConnectionByIdFilter.java | 50 ++++
.../ReadGraphConnectionByTypeFilter.java | 99 ++++++++
.../traverse/ReadGraphConnectionFilter.java | 53 +++++
.../results/ObservableQueryExecutor.java | 2 +-
.../pipeline/cursor/CursorTest.java | 4 +-
67 files changed, 2515 insertions(+), 2160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 7a56631..be52547 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -38,8 +38,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
+import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -180,8 +179,7 @@ public class CpEntityManager implements EntityManager {
private final AsyncEventService indexService;
- private final FilterFactory filterFactory;
- private final CollectorFactory collectorFactory;
+ private final PipelineBuilderFactory filterFactory;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -223,7 +221,7 @@ public class CpEntityManager implements EntityManager {
*/
public CpEntityManager( final CassandraService cass, final CounterUtils counterUtils, final AsyncEventService indexService, final ManagerCache managerCache,
final MetricsFactory metricsFactory, final EntityManagerFig entityManagerFig,
- final FilterFactory filterFactory, final CollectorFactory collectorFactory, final UUID applicationId ) {
+ final PipelineBuilderFactory pipelineBuilderFactory, final UUID applicationId ) {
this.entityManagerFig = entityManagerFig;
@@ -232,10 +230,8 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
Preconditions.checkNotNull( indexService, "indexService must not be null" );
- Preconditions.checkNotNull( filterFactory, "filterFactory must not be null" );
- Preconditions.checkNotNull( collectorFactory, "collectorFactory must not be null" );
- this.filterFactory = filterFactory;
- this.collectorFactory = collectorFactory;
+ Preconditions.checkNotNull( pipelineBuilderFactory, "filterFactory must not be null" );
+ this.filterFactory = pipelineBuilderFactory;
this.managerCache = managerCache;
@@ -750,7 +746,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager relationManager =
- new CpRelationManager( metricsFactory, managerCache, filterFactory, collectorFactory, indexService, this, entityManagerFig, applicationId, entityRef );
+ new CpRelationManager( managerCache, filterFactory, indexService, this, entityManagerFig, applicationId, entityRef );
return relationManager;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 5055538..baa1148 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -35,8 +35,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.ReIndexService;
-import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
+import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
import org.apache.usergrid.persistence.AbstractEntity;
@@ -126,8 +125,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
private final AsyncEventService indexService;
- private final FilterFactory filterFactory;
- private final CollectorFactory collectorFactory;
+ private final PipelineBuilderFactory pipelineBuilderFactory;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector ) {
@@ -141,8 +139,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
this.indexService = injector.getInstance( AsyncEventService.class );
- this.filterFactory = injector.getInstance( FilterFactory.class );
- this.collectorFactory = injector.getInstance( CollectorFactory.class );
+ this.pipelineBuilderFactory = injector.getInstance( PipelineBuilderFactory.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
@@ -203,7 +200,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, entityManagerFig,
- filterFactory, collectorFactory, applicationId );
+
+ pipelineBuilderFactory, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 6201fe8..1c34929 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -31,9 +31,11 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
-import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
-import org.apache.usergrid.corepersistence.pipeline.read.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.builder.EntityBuilder;
+import org.apache.usergrid.corepersistence.pipeline.builder.IdBuilder;
+import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.corepersistence.results.ObservableQueryExecutor;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
@@ -52,7 +54,6 @@ import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
-import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
@@ -121,13 +122,12 @@ public class CpRelationManager implements RelationManager {
private final AsyncEventService indexService;
- private final FilterFactory filterFactory;
- private final CollectorFactory collectorFactory;
+ private final PipelineBuilderFactory pipelineBuilderFactory;
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache,
- final FilterFactory filterFactory, final CollectorFactory collectorFactory, final AsyncEventService indexService,
+ public CpRelationManager( final ManagerCache managerCache,
+ final PipelineBuilderFactory pipelineBuilderFactory, final AsyncEventService indexService,
final EntityManager em, final EntityManagerFig entityManagerFig, final UUID applicationId,
final EntityRef headEntity ) {
@@ -147,8 +147,7 @@ public class CpRelationManager implements RelationManager {
this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
- this.filterFactory = filterFactory;
- this.collectorFactory = collectorFactory;
+ this.pipelineBuilderFactory = pipelineBuilderFactory;
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading head entity {}:{} from app {}", new Object[] {
@@ -629,29 +628,23 @@ public class CpRelationManager implements RelationManager {
query = adjustQuery( query );
- final FilterPipeline<Id> filterPipeline = new FilterPipeline( applicationScope, query.getCursor(), query.getLimit() ).withFilter( filterFactory.getEntityIdFilter( cpHeadEntity.getId() ) );
+ final IdBuilder pipelineBuilder =
+ pipelineBuilderFactory.create( applicationScope ).withCursor( query.getCursor() )
+ .withLimit( query.getLimit() ).fromId( cpHeadEntity.getId() );
- final FilterPipeline<org.apache.usergrid.persistence.model.entity.Entity> entityFilterPipeline;
+ final EntityBuilder results;
if ( query.isGraphSearch() ) {
- entityFilterPipeline = filterPipeline.withFilter( filterFactory.readGraphCollectionFilter( collectionName ) )
- .withFilter( filterFactory.entityLoadFilter() );
+ results = pipelineBuilder.traverseCollection( collectionName ).loadEntities();
}
else {
final String entityType = collection.getType();
-
- entityFilterPipeline = filterPipeline.withFilter(
- filterFactory.elasticSearchCollectionFilter( query.getQl().get(), collectionName, entityType ) )
- .withFilter( filterFactory.candidateEntityFilter() );
+ results = pipelineBuilder.searchCollection( collectionName, entityType, query.getQl().get() ).loadEntities();
}
- final Observable<ResultsPage> resultsObservable =
- entityFilterPipeline.withFilter( filterFactory.entityResumeFilter() )
- .withCollector( collectorFactory.getResultsPageCollector() ).execute();
-
- return new ObservableQueryExecutor( resultsObservable ).next();
+ return new ObservableQueryExecutor( results.build() ).next();
}
@@ -923,7 +916,7 @@ public class CpRelationManager implements RelationManager {
query = adjustQuery( query );
- final String entityType = query.getEntityType();
+ final Optional<String> entityType = Optional.fromNullable( query.getEntityType() ) ;
//set startid -- graph | es query filter -- load entities filter (verifies exists) --> results page collector
// -> 1.0 results
@@ -935,31 +928,57 @@ public class CpRelationManager implements RelationManager {
// collector
- final FilterPipeline<Id> filterPipeline =
- new FilterPipeline( applicationScope, query.getCursor(), query.getLimit() )
- .withFilter( filterFactory.getEntityIdFilter( cpHeadEntity.getId() ) );
+ final IdBuilder
+ pipelineBuilder = pipelineBuilderFactory.create( applicationScope ).withCursor( query.getCursor() ).withLimit( query.getLimit() ).fromId(
+ cpHeadEntity.getId() );
+
+
+
+
+ if(query.getResultsLevel() == Level.REFS){
+ final Observable<ResultsPage<ConnectionRef>> results;
+
+ if(query.isGraphSearch()){
+
+ results = pipelineBuilder.traverseConnection( connection, entityType ).loadConnectionRefs( cpHeadEntity.getId(), connection ).build();
+
+
+ }
+ else
+ {
+ results = pipelineBuilder.searchConnection( connection, query.getQl().get(),entityType) .loadIds().loadConnectionRefs( cpHeadEntity.getId(), connection ).build();
+
+ }
+
+ throw new UnsupportedOperationException( "Implement me" );
+
+ }
+
+
+
+ if(query.getResultsLevel() == Level.IDS){
+
+ throw new UnsupportedOperationException( "Not yet implemented" );
+ }
+
+
+ //we want to load all entities
+ final Observable<ResultsPage<org.apache.usergrid.persistence.model.entity.Entity>> results;
- final FilterPipeline<org.apache.usergrid.persistence.model.entity.Entity> entityFilterPipeline;
if ( query.isGraphSearch() ) {
- entityFilterPipeline = filterPipeline.withFilter( filterFactory.readGraphConnectionFilter( connection ) )
- .withFilter( filterFactory.entityLoadFilter() );
+ results = pipelineBuilder.traverseConnection( connection, entityType ).loadEntities().build();
}
else {
- entityFilterPipeline = filterPipeline.withFilter( filterFactory
- .elasticSearchConnectionFilter( query.getQl().get(), connection, Optional.fromNullable( entityType ) ) )
- .withFilter( filterFactory.candidateEntityFilter() );
+ results = pipelineBuilder.searchConnection( connection, query.getQl().get() , entityType).loadEntities().build();
}
- final Observable<ResultsPage> resultsObservable =
- entityFilterPipeline.withFilter( filterFactory.entityResumeFilter() )
- .withCollector( collectorFactory.getResultsPageCollector() ).execute();
- return new ObservableQueryExecutor( resultsObservable ).next();
+ return new ObservableQueryExecutor( results ).next();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java
new file mode 100644
index 0000000..089f47d
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/FilterPipeline.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+
+import rx.Observable;
+
+
+/**
+ * Pipeline for applying our UG domain specific filters.
+ *
+ * Modeled after an observable, with typing to allow input of specific filters
+ *
+ * @param InputType the input type in the current pipeline state
+ */
+public class FilterPipeline<InputType> {
+
+
+ private int idCount = 0;
+
+ private final ApplicationScope applicationScope;
+
+
+ private final RequestCursor requestCursor;
+ private int limit;
+
+ //Generics hell, intentionally without a generic, we check at the filter level
+ private Observable currentObservable;
+
+
+ /**
+ * Create our filter pipeline
+ */
+ public FilterPipeline( final ApplicationScope applicationScope, final Optional<String> cursor, final int limit ) {
+
+
+ ValidationUtils.validateApplicationScope( applicationScope );
+ Preconditions.checkNotNull( cursor, "cursor optional is required" );
+ Preconditions.checkArgument( limit > 0, "limit must be > 0" );
+
+
+ this.applicationScope = applicationScope;
+
+ //init our cursor to empty
+ this.requestCursor = new RequestCursor( cursor );
+
+ //set the default limit
+ this.limit = limit;
+
+ //set our observable to start at the application
+ final FilterResult<Id> filter = new FilterResult<>( applicationScope.getApplication(), Optional.absent() );
+ this.currentObservable = Observable.just( filter );
+ }
+
+
+ public <OutputType> FilterPipeline<OutputType> withFilter(
+ final PipelineOperation<? super InputType, ? extends OutputType> filter ) {
+
+
+
+ final PipelineContext context = new PipelineContext( applicationScope, requestCursor, limit, idCount );
+
+ filter.setContext( context );
+
+ //done for clarity
+ idCount++;
+
+ return ( FilterPipeline<OutputType> ) this;
+ }
+
+
+
+ /**
+ * Return the observable of the filter pipeline
+ */
+ public Observable<InputType> execute() {
+ return currentObservable;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
index ef696bd..8ec8704 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
@@ -20,7 +20,6 @@
package org.apache.usergrid.corepersistence.pipeline;
-import org.apache.usergrid.corepersistence.pipeline.read.CollectorFactory;
import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
import com.google.inject.AbstractModule;
@@ -44,6 +43,5 @@ public class PipelineModule extends AbstractModule {
//Use Guice to create the builder since we don't really need to do anything
//other than DI when creating the filters
install( new FactoryModuleBuilder().build( FilterFactory.class ) );
- install( new FactoryModuleBuilder().build( CollectorFactory.class ));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperation.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperation.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperation.java
index d2fa16c..3dda22e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperation.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineOperation.java
@@ -33,7 +33,7 @@ import rx.Observable;
* @param <T> The input type of the filter value
* @param <R> The output type of the filter value
*/
-public interface PipelineOperation<T, R> extends Observable.Transformer<FilterResult<T>, R> {
+public interface PipelineOperation<T, R> extends Observable.Transformer<T, R> {
void setContext(final PipelineContext pipelineContext);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java
new file mode 100644
index 0000000..5cb2eab
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/CandidateBuilder.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.builder;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
+import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.corepersistence.pipeline.read.search.Candidate;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class CandidateBuilder {
+
+
+ private final FilterPipeline<FilterResult<Candidate>> filterPipeline;
+ private final FilterFactory filterFactory;
+
+
+ public CandidateBuilder( final FilterPipeline<FilterResult<Candidate>> filterPipeline,
+ final FilterFactory filterFactory ) {
+ this.filterPipeline = filterPipeline;
+ this.filterFactory = filterFactory;
+ }
+
+
+ /**
+ * Validates all candidates for the versions by id and sets them
+ * @return
+ */
+ public IdBuilder loadIds(){
+
+ final FilterPipeline<FilterResult<Id>> newFilter = filterPipeline.withFilter( filterFactory.candidateResultsIdVerifyFilter() );
+
+ return new IdBuilder( newFilter, filterFactory );
+ }
+
+
+ /**
+ * Load all the candidates as entities and return them
+ * @return
+ */
+ public EntityBuilder loadEntities(){
+
+ final FilterPipeline<FilterResult<Entity>> newFilter = filterPipeline.withFilter( filterFactory.candidateEntityFilter() );
+
+ return new EntityBuilder(newFilter );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java
new file mode 100644
index 0000000..b4ea94e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionBuilder.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.builder;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.persistence.ConnectionRef;
+
+import rx.Observable;
+
+
+public class ConnectionBuilder {
+
+
+
+ public Observable<ResultsPage<ConnectionRef>> build(){
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java
new file mode 100644
index 0000000..6c0ebc8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/ConnectionRefBuilder.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.builder;
+
+
+import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.ResultsPageCollector;
+import org.apache.usergrid.persistence.ConnectionRef;
+
+import rx.Observable;
+
+
+/**
+ * A 1.0 compatibility state. Should be removed as services are refactored
+ */
+@Deprecated
+public class ConnectionRefBuilder {
+
+
+ private final FilterPipeline<FilterResult<ConnectionRef>> connectionRefFilter;
+
+ public ConnectionRefBuilder( final FilterPipeline<FilterResult<ConnectionRef>> connectionRefFilter ) {
+ this.connectionRefFilter = connectionRefFilter;
+ }
+
+
+ /**
+ * Build our connection refs observable
+ * @return
+ */
+ public Observable<ResultsPage<ConnectionRef>> build(){
+ return connectionRefFilter.withFilter( new ResultsPageCollector<>() ).execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java
new file mode 100644
index 0000000..07b4586
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/EntityBuilder.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.builder;
+
+
+import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+import rx.Observable;
+
+
+/**
+ * Builder to build our entity state
+ */
+public class EntityBuilder {
+
+ private final FilterPipeline<FilterResult<Entity>> filterPipeline;
+
+
+ public EntityBuilder( final FilterPipeline<FilterResult<Entity>> filterPipeline ) {
+ this.filterPipeline = filterPipeline;
+ }
+
+
+ /**
+ * Build our results of entities
+ * @return
+ */
+ public Observable<ResultsPage<Entity>> build(){
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
new file mode 100644
index 0000000..12a89ba
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.builder;
+
+
+import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
+import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefResumeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.search.Candidate;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionByTypeFilter;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A builder to transition from emitting Ids in the pipeline into other operations
+ */
+public class IdBuilder {
+
+
+ private final FilterFactory filterFactory;
+ private final FilterPipeline<FilterResult<Id>> filterPipeline;
+
+
+ public IdBuilder( final FilterPipeline<FilterResult<Id>> filterPipeline, final FilterFactory filterFactory ) {
+ this.filterPipeline = filterPipeline;
+ this.filterFactory = filterFactory;
+ }
+
+
+ /**
+ * Load all the ids we encounter when traversing the graph as entities
+ * @return
+ */
+ public EntityBuilder loadEntities() {
+ final FilterPipeline<FilterResult<Entity>> pipeline =
+ filterPipeline.withFilter( filterFactory.entityLoadFilter() );
+
+ return new EntityBuilder( pipeline );
+ }
+
+
+ /**
+ * Traverse all the collection edges from our input Id
+ * @param collectionName
+ * @return
+ */
+ public IdBuilder traverseCollection( final String collectionName ) {
+ final FilterPipeline<FilterResult<Id>> newFilter =
+ filterPipeline.withFilter( filterFactory.readGraphCollectionFilter( collectionName ) );
+
+ return new IdBuilder( newFilter, filterFactory );
+ }
+
+
+ /**
+ * Traverse all connection edges from our input Id
+ * @param connectionName The name of the connection
+ * @param entityType The optional type of the entity
+ * @return
+ */
+ public IdBuilder traverseConnection( final String connectionName, final Optional<String> entityType ) {
+
+ final PipelineOperation<FilterResult<Id>, FilterResult<Id>> filter;
+
+ if(entityType.isPresent()){
+ filter = filterFactory.readGraphConnectionByTypeFilter( connectionName, entityType.get() );
+ }else{
+ filter = filterFactory.readGraphConnectionFilter( connectionName );
+ }
+
+
+ return new IdBuilder( filterPipeline.withFilter(filter ), filterFactory );
+ }
+
+
+ /**
+ * Search all collections from our inputId with the specified criteria
+ * @param collectionName The name of the collection
+ * @param ql The user's query to execute
+ * @param entityType The type of the entity
+ * @return Candidate results
+ */
+ public CandidateBuilder searchCollection( final String collectionName, final String ql, final String entityType ) {
+
+ final FilterPipeline<FilterResult<Candidate>> newFilter = filterPipeline.withFilter( filterFactory.searchCollectionFilter(
+ ql, collectionName, entityType ) );
+
+ return new CandidateBuilder( newFilter, filterFactory );
+ }
+
+
+ /**
+ * Search all connections from our input Id and search their connections
+ * @param connectionName The connection name to search
+ * @param ql The query to execute
+ * @param entityType The optional type of entity. If this is absent, all entity types in the connection will be searched
+ * @return Candidate results
+ */
+ public CandidateBuilder searchConnection( final String connectionName, final String ql , final Optional<String> entityType) {
+
+
+ final FilterPipeline<FilterResult<Candidate>> newFilter = filterPipeline.withFilter( filterFactory.searchConnectionFilter(
+ ql, connectionName, entityType ) );
+
+ return new CandidateBuilder( newFilter, filterFactory );
+ }
+
+
+ /**
+ * Create connection refs from our ids. This is a legacy operation
+ * @param sourceId
+ * @param connectionType
+ * @return
+ */
+ @Deprecated
+ public ConnectionRefBuilder loadConnectionRefs(final Id sourceId, final String connectionType){
+
+ final FilterPipeline<FilterResult<ConnectionRef>> connectionRefFilter = filterPipeline.withFilter( new ConnectionRefFilter(sourceId, connectionType ) ).withFilter(
+ new ConnectionRefResumeFilter() );
+ return new ConnectionRefBuilder(connectionRefFilter);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java
new file mode 100644
index 0000000..488e9c1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilder.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.builder;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
+import org.apache.usergrid.corepersistence.pipeline.FilterPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+
+/**
+ * This is our root builder to build filter pipelines. All operations should start with an instance of this class, and compose
+ * graph operations by traversing various builders to create our filter pipeline
+ */
+public class PipelineBuilder {
+
+
+
+ private final ApplicationScope applicationScope;
+ private Optional<String> cursor = Optional.absent();
+ private int limit = 10;
+ private final FilterFactory filterFactory;
+
+
+ /**
+ * Create an instance of our I/O operations
+ * @param filterFactory
+ */
+ @Inject
+ public PipelineBuilder( final FilterFactory filterFactory, @Assisted final ApplicationScope applicationScope ) {
+ this.filterFactory = filterFactory;
+ this.applicationScope = applicationScope;
+ }
+
+
+
+
+ /**
+ * Set the cursor to use in our filter pipline
+ * @param cursor
+ * @return
+ */
+ public PipelineBuilder withCursor(final Optional<String> cursor){
+ Preconditions.checkNotNull(cursor, "cursor must not be null");
+ this.cursor = cursor;
+ return this;
+ }
+
+
+ /**
+ * Set our limit
+ * @param limit
+ * @return
+ */
+ public PipelineBuilder withLimit(final int limit){
+ this.limit = limit;
+ return this;
+ }
+
+
+ /**
+ * Set our start point in our graph traversal to the specified entity id. A 1.0 compatibility API. eventually this should be replaced with
+ * a call that will allow us to start traversing at the application node to any other node in the graph
+ *
+ * @param entityId
+ * @return
+ */
+ @Deprecated
+ public IdBuilder fromId(final Id entityId){
+ FilterPipeline<FilterResult<Id>> filterPipeline = new FilterPipeline( applicationScope, this.cursor,limit ).withFilter( filterFactory.getEntityIdFilter( entityId ) );
+
+ return new IdBuilder( filterPipeline, filterFactory );
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java
new file mode 100644
index 0000000..6cb515b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/PipelineBuilderFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.builder;
+
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+
+public interface PipelineBuilderFactory {
+
+
+ /**
+ * Create our pipeline builder to allow us to build our pipeline
+ * @param applicationScope
+ * @return
+ */
+ PipelineBuilder create( final ApplicationScope applicationScope );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
index e4d5d44..64cf67f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
@@ -29,7 +29,7 @@ import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
* @param <T> the input type
* @param <R> The output Type
*/
-public abstract class AbstractFilter<T, R> implements Filter<T, R> {
+public abstract class AbstractFilter<T, R> implements PipelineOperation<T, R> {
protected PipelineContext pipelineContext;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
index c68dc4a..6dc4561 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
@@ -33,7 +33,7 @@ import com.google.common.base.Optional;
* @param <R> The response type
* @param <C> The cursor type
*/
-public abstract class AbstractPathFilter<T, R, C extends Serializable> extends AbstractFilter<T, R> implements Filter<T, R> {
+public abstract class AbstractPathFilter<T, R, C extends Serializable> extends AbstractFilter<FilterResult<T>, FilterResult<R>> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Collector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Collector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Collector.java
deleted file mode 100644
index e28ce44..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Collector.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
-
-
-/**
- * A command that is used to reduce our stream of results into a stream of final batch outputs. When used
- * no further transformation or encoding should occur. Otherwise EdgePath data will be lost, and serialization cannot occur
- * across requests
- *
- * @param <T> The input type
- * @param <R> The output type
- */
-public interface Collector<T, R> extends PipelineOperation<T,R> {
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFactory.java
deleted file mode 100644
index dd200b5..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import org.apache.usergrid.corepersistence.pipeline.read.collect.ResultsPageCollector;
-
-
-/**
- * A factory for generating collectors
- */
-public interface CollectorFactory {
-
-
- /**
- * Get the results page collector
- * @return
- */
- ResultsPageCollector getResultsPageCollector();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
deleted file mode 100644
index ee01602..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
-
-
-/**
- * Traverses edges in the graph. Either by query or graph traversal. Take an observable of FilterResult, and emits
- * an observable of FilterResults. Filters should never emit groups or objects that represent collections. Items should
- * always be emitted 1 at a time. It is the responsibility of the collector to aggregate results.
- */
-public interface Filter<T, R> extends PipelineOperation<T, FilterResult<R>> {
-
- /**
- * Get the builder for the next phase
- * @return
- */
-// B getNextBuilder();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
index d297c2a..ca5695c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
@@ -20,18 +20,20 @@
package org.apache.usergrid.corepersistence.pipeline.read;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefResumeFilter;
import org.apache.usergrid.corepersistence.pipeline.read.collect.EntityResumeFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.CandidateEntityFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.CandidateIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.ElasticSearchCollectionFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.ElasticSearchConnectionFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.EntityIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.EntityLoadFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionByIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByTypeFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.search.CandidateEntityFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.search.CandidateIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.search.SearchCollectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.search.SearchConnectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.EntityIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.EntityLoadVerifyFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphCollectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphCollectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionByTypeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.ReadGraphConnectionFilter;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
@@ -90,10 +92,9 @@ public interface FilterFactory {
* @param query The query to use when querying the entities in the collection
* @param collectionName The collection name to use when querying
*/
- ElasticSearchCollectionFilter elasticSearchCollectionFilter( @Assisted( "query" ) final String query,
- @Assisted( "collectionName" )
- final String collectionName,
- @Assisted( "entityType" ) final String entityType );
+ SearchCollectionFilter searchCollectionFilter( @Assisted( "query" ) final String query,
+ @Assisted( "collectionName" ) final String collectionName,
+ @Assisted( "entityType" ) final String entityType );
/**
@@ -103,17 +104,16 @@ public interface FilterFactory {
* @param connectionName The type of connection to query
* @param connectedEntityType The type of entity in the connection. Leave absent to query all entity types
*/
- ElasticSearchConnectionFilter elasticSearchConnectionFilter( @Assisted( "query" ) final String query,
- @Assisted( "connectionName" )
- final String connectionName,
- @Assisted( "connectedEntityType" )
- final Optional<String> connectedEntityType );
+ SearchConnectionFilter searchConnectionFilter( @Assisted( "query" ) final String query,
+ @Assisted( "connectionName" ) final String connectionName,
+ @Assisted( "connectedEntityType" )
+ final Optional<String> connectedEntityType );
/**
* Generate a new instance of the command with the specified parameters
*/
- EntityLoadFilter entityLoadFilter();
+ EntityLoadVerifyFilter entityLoadFilter();
/**
* Get the collector for collection candidate results to entities
@@ -127,16 +127,37 @@ public interface FilterFactory {
CandidateIdFilter candidateResultsIdVerifyFilter();
/**
- * Get an entity id filter. Used as a 1.0->2.0 bridge since we're not doing full traversals
- *
* @param entityId The entity id to emit
+ *
+ * @deprecated A 1.0 api
+ *
+ * Get an entity id filter. Used as a 1.0->2.0 bridge since we're not doing full traversals
*/
+ @Deprecated
EntityIdFilter getEntityIdFilter( final Id entityId );
/**
* Create a new instance of our entity filter
- * @return
*/
EntityResumeFilter entityResumeFilter();
+
+ /**
+ * @deprecated A 1.0 api Create a filter for resuming connection references
+ */
+ @Deprecated
+ ConnectionRefResumeFilter connectionRefResumeFilter();
+
+ /**
+ *
+ * Creates connection refs for 1.0 compatibilty
+ *
+ * @param sourceId The source id
+ * @param connectionType The connection type
+ *
+ * @deprecated A 1.0 api Create a filter for transforming incoming ids into connection refs
+ */
+ @Deprecated
+ ConnectionRefFilter connectionRefFilter( @Assisted( "sourceId" ) final Id sourceId,
+ @Assisted( "connectionType" ) final String connectionType );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java
deleted file mode 100644
index f8bbdd8..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterPipeline.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import org.apache.usergrid.corepersistence.pipeline.PipelineContext;
-import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
-import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Preconditions;
-
-import rx.Observable;
-
-
-/**
- * Pipeline for applying our UG domain specific filters.
- *
- * Modeled after an observable, with typing to allow input of specific filters
- *
- * @param InputType the input type in the current pipeline state
- */
-public class FilterPipeline<InputType> {
-
-
- private int idCount = 0;
-
- private final ApplicationScope applicationScope;
-
-
- private final RequestCursor requestCursor;
- private int limit;
-
- //Generics hell, intentionally without a generic, we check at the filter level
- private Observable currentObservable;
-
-
- /**
- * Create our filter pipeline
- */
- public FilterPipeline( final ApplicationScope applicationScope, final Optional<String> cursor, final int limit ) {
-
-
- ValidationUtils.validateApplicationScope( applicationScope );
- Preconditions.checkNotNull( cursor, "cursor optional is required" );
- Preconditions.checkArgument( limit > 0, "limit must be > 0" );
-
-
- this.applicationScope = applicationScope;
-
- //init our cursor to empty
- this.requestCursor = new RequestCursor( cursor );
-
- //set the default limit
- this.limit = limit;
-
- //set our observable to start at the application
- final FilterResult<Id> filter = new FilterResult<>( applicationScope.getApplication(), Optional.absent() );
- this.currentObservable = Observable.just( filter );
- }
-
-
- public <OutputType> FilterPipeline<OutputType> withFilter(
- final Filter<? super InputType, ? extends OutputType> filter ) {
-
-
- setUp( filter );
-
- return ( FilterPipeline<OutputType> ) this;
- }
-
-
- public <OutputType> FilterPipeline<OutputType> withCollector(
- final Collector<? super InputType, ? extends OutputType> collector ) {
-
-
- setUp( collector );
-
- return ( FilterPipeline<OutputType> ) this;
- }
-
-
- private <OutputType> void setUp(
- final PipelineOperation<? super InputType, ? extends OutputType> pipelineOperation ) {
- setState( pipelineOperation );
-
- currentObservable = currentObservable.compose( pipelineOperation );
- }
-
-
- /**
- * Return the observable of the filter pipeline
- */
- public Observable<InputType> execute() {
- return currentObservable;
- }
-
-
- /**
- * Set the id of the state
- */
- private void setState( final PipelineOperation pipelineOperation ) {
-
-
- final PipelineContext context = new PipelineContext( applicationScope, requestCursor, limit, idCount );
-
- pipelineOperation.setContext( context );
-
- //done for clarity
- idCount++;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java
deleted file mode 100644
index 0f73fb9..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import com.google.inject.Singleton;
-
-
-@Singleton
-public class ReadFilterFactoryImpl { //implements ReadFilterFactory {
-
-//
-// private final GraphManagerFactory graphManagerFactory;
-// private final EntityIndexFactory entityIndexFactory;
-// private final EntityCollectionManagerFactory entityCollectionManagerFactory;
-//
-//
-// @Inject
-// public ReadFilterFactoryImpl( final GraphManagerFactory graphManagerFactory,
-// final EntityIndexFactory entityIndexFactory,
-// final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
-//
-//
-// this.graphManagerFactory = graphManagerFactory;
-// this.entityIndexFactory = entityIndexFactory;
-// this.entityCollectionManagerFactory = entityCollectionManagerFactory;
-// }
-//
-//
-// @Override
-// public ReadGraphCollectionFilter readGraphCollectionCommand( final String collectionName ) {
-// return new ReadGraphCollectionFilter( graphManagerFactory, collectionName );
-// }
-//
-//
-// @Override
-// public ReadGraphCollectionByIdFilter readGraphCollectionByIdFilter( final String collectionName,
-// final Id targetId ) {
-// return new ReadGraphCollectionByIdFilter( graphManagerFactory, collectionName, targetId );
-// }
-//
-//
-// @Override
-// public ReadGraphConnectionFilter readGraphConnectionCommand( final String connectionName ) {
-// return new ReadGraphConnectionFilter( graphManagerFactory, connectionName );
-// }
-//
-//
-// @Override
-// public ReadGraphConnectionByTypeFilter readGraphConnectionCommand( final String connectionName,
-// final String entityType ) {
-// return new ReadGraphConnectionByTypeFilter( graphManagerFactory, connectionName, entityType );
-// }
-//
-//
-// @Override
-// public ReadGraphConnectionByIdFilter readGraphConnectionByIdFilter( final String connectionName,
-// final Id targetId ) {
-// return new ReadGraphConnectionByIdFilter( graphManagerFactory, connectionName, targetId );
-// }
-//
-//
-// @Override
-// public EntityLoadCollector entityLoadCollector() {
-// return new EntityLoadCollector( entityCollectionManagerFactory );
-// }
-//
-//
-// /**
-// * TODO refactor these impls to use RX internally, as well as remove the query object
-// */
-// @Override
-// public QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector(
-// final String collectionName, final String query ) {
-//
-// final Query queryObject = Query.fromQL( query );
-//
-// final QueryCollectionElasticSearchCollectorFilter filter =
-// new QueryCollectionElasticSearchCollectorFilter( entityCollectionManagerFactory, entityIndexFactory,
-// collectionName, queryObject );
-//
-// return filter;
-// }
-//
-//
-// @Override
-// public QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector(
-// final String connectionName, final String query ) {
-//
-// final Query queryObject = Query.fromQL( query );
-//
-// final QueryConnectionElasticSearchCollectorFilter filter =
-// new QueryConnectionElasticSearchCollectorFilter( entityCollectionManagerFactory, entityIndexFactory,
-// connectionName, queryObject );
-//
-// return filter;
-// }
-//
-//
-// @Override
-// public QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector(
-// final String connectionName, final String connectionEntityType, final String query ) {
-//
-// final Query queryObject = Query.fromQL( query );
-// queryObject.setConnectionType( connectionEntityType );
-//
-// final QueryConnectionElasticSearchCollectorFilter filter =
-// new QueryConnectionElasticSearchCollectorFilter( entityCollectionManagerFactory, entityIndexFactory,
-// connectionName, queryObject );
-//
-// return filter;
-// }
-//
-//
-// @Override
-// public EntityIdFilter getEntityIdFilter( final Id entityId ) {
-// return new EntityIdFilter( entityId );
-// }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
index 1810d65..6b3a086 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ResultsPage.java
@@ -29,24 +29,26 @@ import org.apache.usergrid.persistence.model.entity.Entity;
/**
* An encapsulation of entities as a group of responses. Ordered by the requesting filters. Each set should be
* considered a "page" of results. A hold over from 1.0. We shouldn't need this when we fully move away from the EM/RM
+ *
+ * @param T the type of results page
*/
-public class ResultsPage {
+public class ResultsPage<T> {
- private final List<Entity> entityList;
+ private final List<T> entityList;
private final int limit;
private final ResponseCursor responseCursor;
- public ResultsPage( final List<Entity> entityList, final ResponseCursor responseCursor, final int limit ) {
+ public ResultsPage( final List<T> entityList, final ResponseCursor responseCursor, final int limit ) {
this.entityList = entityList;
this.responseCursor = responseCursor;
this.limit = limit;
}
- public List<Entity> getEntityList() {
+ public List<T> getEntityList() {
return entityList;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/AbstractCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/AbstractCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/AbstractCollector.java
deleted file mode 100644
index 1c5175d..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/AbstractCollector.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read.collect;
-
-
-import org.apache.usergrid.corepersistence.pipeline.PipelineContext;
-import org.apache.usergrid.corepersistence.pipeline.read.Collector;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
-
-
-/**
- * Basic functionality for our commands to handle cursor IO
- * @param <T> the input type
- * @param <R> The output Type
- */
-public abstract class AbstractCollector<T, R> implements Collector<T, R> {
-
-
- protected PipelineContext pipelineContext;
-
-
- @Override
- public void setContext( final PipelineContext pipelineContext ) {
- this.pipelineContext = pipelineContext;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefFilter.java
new file mode 100644
index 0000000..392e33a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefFilter.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.collect;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+
+/**
+ * This class only exists for 1.0 compatibility, remove once services no longer need connection refs
+ */
+public class ConnectionRefFilter extends AbstractFilter<FilterResult<Id>, FilterResult<ConnectionRef>> {
+
+
+ private final Id sourceId;
+ private final String connectionType;
+
+
+ @Inject
+ public ConnectionRefFilter( @Assisted( "sourceId" ) final Id sourceId,
+ @Assisted( "connectionType" ) final String connectionType ) {
+ this.sourceId = sourceId;
+ this.connectionType = connectionType;
+ }
+
+
+ @Override
+ public Observable<FilterResult<ConnectionRef>> call( final Observable<FilterResult<Id>> filterResultObservable ) {
+
+ return filterResultObservable.map( targetResult -> {
+
+ final Id targetId = targetResult.getValue();
+ final ConnectionRef ref =
+ new ConnectionRefImpl( sourceId.getType(), sourceId.getUuid(), connectionType, targetId.getType(),
+ targetId.getUuid() );
+
+ return new FilterResult<>( ref, Optional.<EdgePath>absent() );
+ } );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefResumeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefResumeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefResumeFilter.java
new file mode 100644
index 0000000..5c3a93e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/ConnectionRefResumeFilter.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.collect;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.ConnectedEntityRef;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * A filter that is used when we can potentially serialize pages via cursor. This will filter the first result, only if
+ * it matches the Id that was set. This is a 1.0 compatibility implementation, and should be removed when services no
+ * longer depends on connection refs
+ */
+public class ConnectionRefResumeFilter extends AbstractPathFilter<ConnectionRef, ConnectionRef, Id> {
+
+
+ @Override
+ public Observable<FilterResult<ConnectionRef>> call(
+ final Observable<FilterResult<ConnectionRef>> filterResultObservable ) {
+
+ //filter only the first id, then map into our path for our next pass
+
+
+ return filterResultObservable.skipWhile( filterResult -> {
+
+ final Optional<Id> startFromCursor = getSeekValue();
+
+
+ if ( !startFromCursor.isPresent() ) {
+ return false;
+ }
+
+ final ConnectedEntityRef ref = filterResult.getValue().getTargetRefs();
+
+ final Id entityId = startFromCursor.get();
+
+ return entityId.getUuid().equals( ref.getUuid() ) && entityId.getType().equals( ref.getType() );
+ } ).map( filterResult -> {
+
+
+ final ConnectionRef entity = filterResult.getValue();
+
+ final String type = entity.getTargetRefs().getType();
+ final UUID uuid = entity.getTargetRefs().getUuid();
+
+ final Id entityId = new SimpleId( uuid, type );
+
+ return createFilterResult( entity, entityId, filterResult.getPath() );
+ } );
+ }
+
+
+ @Override
+ protected CursorSerializer<Id> getCursorSerializer() {
+ return IdCursorSerializer.INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java
index 2917b61..f545631 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/EntityResumeFilter.java
@@ -22,7 +22,6 @@ package org.apache.usergrid.corepersistence.pipeline.read.collect;
import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.Filter;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -36,7 +35,7 @@ import rx.Observable;
* A filter that is used when we can potentially serialize pages via cursor. This will filter the first result, only if
* it matches the Id that was set
*/
-public class EntityResumeFilter extends AbstractPathFilter<Entity, Entity, Id> implements Filter<Entity, Entity> {
+public class EntityResumeFilter extends AbstractPathFilter<Entity, Entity, Id> {
@Override
[12/12] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
USERGRID-608
Posted by gr...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-608
# By Todd Nine (5) and Shawn Feldman (2)
# Via Shawn Feldman (2) and Todd Nine (1)
* 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid:
observable changes
add is application method
Updated bi-directional to create 2 directional indexes
Finishes testing of connections
Updates pipeline and fixes connectionref querying
Refactors operations into easier build pattern. Pipeline still need some work.
Refactor of pipeline to support type mapping for clarity
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/87963740
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/87963740
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/87963740
Branch: refs/heads/USERGRID-608
Commit: 87963740a16c796710c64cb0f67b3195992e9170
Parents: 29a4009 1396ebe
Author: GERey <gr...@apigee.com>
Authored: Tue May 26 16:06:17 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Tue May 26 16:06:17 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 24 +-
.../corepersistence/CpEntityManagerFactory.java | 3 +-
.../corepersistence/CpRelationManager.java | 219 ++++++++------
.../corepersistence/index/IndexServiceImpl.java | 57 +---
.../corepersistence/pipeline/Pipeline.java | 93 +++---
.../pipeline/PipelineBuilderFactory.java | 39 ---
.../pipeline/PipelineModule.java | 20 +-
.../pipeline/PipelineOperation.java | 2 +-
.../pipeline/PipelineOperations.java | 30 ++
.../pipeline/builder/CandidateBuilder.java | 67 +++++
.../pipeline/builder/ConnectionRefBuilder.java | 53 ++++
.../pipeline/builder/EntityBuilder.java | 54 ++++
.../pipeline/builder/IdBuilder.java | 151 ++++++++++
.../pipeline/builder/PipelineBuilder.java | 100 +++++++
.../builder/PipelineBuilderFactory.java | 35 +++
.../pipeline/read/AbstractFilter.java | 2 +-
.../pipeline/read/AbstractPathFilter.java | 2 +-
.../pipeline/read/Collector.java | 38 ---
.../pipeline/read/CollectorFactory.java | 38 ---
.../corepersistence/pipeline/read/Filter.java | 31 --
.../pipeline/read/FilterFactory.java | 74 +++--
.../pipeline/read/ReadFilterFactoryImpl.java | 136 ---------
.../pipeline/read/ReadPipelineBuilder.java | 104 -------
.../pipeline/read/ReadPipelineBuilderImpl.java | 296 -------------------
.../pipeline/read/ResultsPage.java | 10 +-
.../read/collect/AbstractCollector.java | 46 ---
.../read/collect/ConnectionRefFilter.java | 68 +++++
.../read/collect/ConnectionRefResumeFilter.java | 86 ++++++
.../pipeline/read/collect/EntityFilter.java | 68 -----
.../read/collect/EntityResumeFilter.java | 67 +++++
.../pipeline/read/collect/IdResumeFilter.java | 61 ++++
.../read/collect/ResultsPageCollector.java | 35 ++-
.../AbstractElasticSearchFilter.java | 171 -----------
.../pipeline/read/elasticsearch/Candidate.java | 55 ----
.../elasticsearch/CandidateEntityFilter.java | 234 ---------------
.../read/elasticsearch/CandidateIdFilter.java | 201 -------------
.../ElasticSearchCollectionFilter.java | 77 -----
.../ElasticSearchConnectionFilter.java | 73 -----
.../ElasticsearchCursorSerializer.java | 42 ---
.../read/elasticsearch/Elasticsearchdiagram.jpg | Bin 316655 -> 0 bytes
.../graph/AbstractReadGraphEdgeByIdFilter.java | 82 -----
.../read/graph/AbstractReadGraphFilter.java | 147 ---------
.../read/graph/EdgeCursorSerializer.java | 42 ---
.../pipeline/read/graph/EntityIdFilter.java | 54 ----
.../pipeline/read/graph/EntityLoadFilter.java | 155 ----------
.../pipeline/read/graph/GraphDiagram.jpg | Bin 347711 -> 0 bytes
.../graph/ReadGraphCollectionByIdFilter.java | 49 ---
.../read/graph/ReadGraphCollectionFilter.java | 53 ----
.../graph/ReadGraphConnectionByIdFilter.java | 50 ----
.../graph/ReadGraphConnectionByTypeFilter.java | 100 -------
.../read/graph/ReadGraphConnectionFilter.java | 53 ----
.../search/AbstractElasticSearchFilter.java | 169 +++++++++++
.../pipeline/read/search/Candidate.java | 55 ++++
.../read/search/CandidateEntityFilter.java | 232 +++++++++++++++
.../pipeline/read/search/CandidateIdFilter.java | 190 ++++++++++++
.../search/ElasticsearchCursorSerializer.java | 40 +++
.../read/search/Elasticsearchdiagram.jpg | Bin 0 -> 316655 bytes
.../read/search/SearchCollectionFilter.java | 77 +++++
.../read/search/SearchConnectionFilter.java | 72 +++++
.../AbstractReadGraphEdgeByIdFilter.java | 82 +++++
.../read/traverse/AbstractReadGraphFilter.java | 146 +++++++++
.../read/traverse/EdgeCursorSerializer.java | 42 +++
.../pipeline/read/traverse/EntityIdFilter.java | 53 ++++
.../read/traverse/EntityLoadVerifyFilter.java | 154 ++++++++++
.../pipeline/read/traverse/GraphDiagram.jpg | Bin 0 -> 347711 bytes
.../traverse/ReadGraphCollectionByIdFilter.java | 49 +++
.../traverse/ReadGraphCollectionFilter.java | 53 ++++
.../traverse/ReadGraphConnectionByIdFilter.java | 50 ++++
.../ReadGraphConnectionByTypeFilter.java | 99 +++++++
.../traverse/ReadGraphConnectionFilter.java | 53 ++++
.../results/ConnectionRefQueryExecutor.java | 60 ++++
.../results/EntityQueryExecutor.java | 84 ++++++
.../results/ObservableQueryExecutor.java | 52 ++--
.../corepersistence/results/QueryExecutor.java | 1 +
.../corepersistence/util/CpNamingUtils.java | 5 +
.../pipeline/cursor/CursorTest.java | 4 +-
76 files changed, 2832 insertions(+), 2737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/87963740/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 2166827,e3bbf23..6d0e772
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -167,9 -179,7 +181,9 @@@ public class CpEntityManager implement
private final AsyncEventService indexService;
- private PipelineBuilderFactory pipelineBuilderFactory;
- private final PipelineBuilderFactory filterFactory;
++ private final PipelineBuilderFactory pipelineBuilderFactory;
+
+ private final GraphManagerFactory graphManagerFactory;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@@ -211,8 -221,7 +225,9 @@@
*/
public CpEntityManager( final CassandraService cass, final CounterUtils counterUtils, final AsyncEventService indexService, final ManagerCache managerCache,
final MetricsFactory metricsFactory, final EntityManagerFig entityManagerFig,
- final PipelineBuilderFactory pipelineBuilderFactory, final UUID applicationId ) {
+ final PipelineBuilderFactory pipelineBuilderFactory ,
+ final GraphManagerFactory graphManagerFactory,final UUID applicationId ) {
++
this.entityManagerFig = entityManagerFig;
@@@ -221,12 -230,10 +236,13 @@@
Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
Preconditions.checkNotNull( indexService, "indexService must not be null" );
- Preconditions.checkNotNull( pipelineBuilderFactory, "filterFactory must not be null" );
- this.filterFactory = pipelineBuilderFactory;
+ Preconditions.checkNotNull( pipelineBuilderFactory, "pipelineBuilderFactory must not be null" );
+ Preconditions.checkNotNull( graphManagerFactory, "graphManagerFactory must not be null" );
+ this.pipelineBuilderFactory = pipelineBuilderFactory;
+ this.graphManagerFactory = graphManagerFactory;
+
+
this.managerCache = managerCache;
this.applicationId = applicationId;
this.indexService = indexService;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/87963740/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b11973e,c9e35f0..5b4af6d
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -201,8 -198,10 +201,9 @@@ public class CpEntityManagerFactory imp
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, entityManagerFig,
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache,
+ metricsFactory, entityManagerFig, pipelineBuilderFactory, graphManagerFactory, applicationId );
+
-
- pipelineBuilderFactory, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/87963740/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index d7ba7e4,b57ea92..1f7d4de
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@@ -176,123 -159,26 +169,77 @@@ public class IndexServiceImpl implement
public Observable<IndexOperationMessage> deleteIndexEdge( final ApplicationScope applicationScope,
final Edge edge ) {
+ final Observable<IndexOperationMessage> batches =
+ Observable.just( edge ).flatMap( edgeValue -> {
+ final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( applicationScope );
+ EntityIndexBatch batch = ei.createBatch();
- //TODO, query ES and remove this edge
- throw new NotImplementedException( "Implement me" );
- }
+ //review why generating the Scope from the Source and the target node makes sense.
+ final IndexEdge fromSource = generateScopeFromSource( edge );
+ final Id targetId = edge.getTargetNode();
+
+ CandidateResults targetEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromSource, targetId );
+
+
+ //1. Feed the observable the candidate results you got back. Since it now does the aggregation for you
+ // you don't need to worry about putting your code in a do while.
+ batch = deindexBatchIteratorResolver( fromSource, targetEdgesToBeDeindexed, batch );
-
-
-
++
+ final IndexEdge fromTarget = generateScopeFromTarget( edge );
+ final Id sourceId = edge.getSourceNode();
+
+ CandidateResults sourceEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromTarget, sourceId );
+
+ batch = deindexBatchIteratorResolver( fromTarget, sourceEdgesToBeDeindexed, batch );
+
+ return batch.execute();
+ } );
+
+ return ObservableTimer.time( batches, addTimer );
+ }
+
+ //This should look up the entityId and delete any documents with a timestamp that comes before
+ //The edges that are connected will be compacted away from the graph.
@Override
public Observable<IndexOperationMessage> deleteEntityIndexes( final ApplicationScope applicationScope,
- final Id entityId ) {
+ final Id entityId, final UUID markedVersion ) {
- //TODO query ES and remove this entityId
- throw new NotImplementedException( "Implement me" );
- }
+ //bootstrap the lower modules from their caches
+ final ApplicationEntityIndex ei = entityIndexFactory.createApplicationEntityIndex( applicationScope );
+ CandidateResults crs = ei.getAllEntityVersionsBeforeMarkedVersion( entityId, markedVersion );
+ //not actually sure about the timestamp but ah well. works.
+ SearchEdge searchEdge = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(),
+ CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId,
+ entityId.getUuid().timestamp() ) );
+ final Observable<IndexOperationMessage> batches = Observable.from( crs )
+ //collect results into a single batch
+ .collect( () -> ei.createBatch(), ( batch, candidateResult ) -> {
+ logger.debug( "Deindexing on edge {} for entity {} added to batch",searchEdge , entityId );
+ batch.deindex( searchEdge, candidateResult );
+ } )
+ //return the future from the batch execution
+ .flatMap( batch -> batch.execute() );
+ return ObservableTimer.time(batches, indexTimer);
+ }
-
- /**
- * Get index edges to the target. Used in only certain entity types, such as roles, users, groups etc
- * where we doubly index on both directions of the edge
- *
- * @param graphManager The graph manager
- * @param entityId The entity's id
- */
- private Observable<IndexEdge> getIndexEdgesAsTarget( final GraphManager graphManager, final Id entityId ) {
-
- final String collectionName = InflectionUtils.pluralize( entityId.getType() );
-
-
- final CollectionInfo collection = getDefaultSchema().getCollection( Application.ENTITY_TYPE, collectionName );
-
- //nothing to do
- if ( collection == null ) {
- return Observable.empty();
- }
-
-
- final String linkedCollection = collection.getLinkedCollection();
-
- /**
- * Nothing to link
- */
- if ( linkedCollection == null ) {
- return Observable.empty();
- }
-
-
- /**
- * An observable of sizes as we execute batches
- *
- * we're indexing from target->source here
- */
- return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection )
- .map( edge -> generateScopeFromTarget( edge ) );
- }
-
+ /**
+ * Takes in candidate results and uses the iterator to create batch commands
+ */
+ public EntityIndexBatch deindexBatchIteratorResolver(IndexEdge edge,CandidateResults edgesToBeDeindexed, EntityIndexBatch batch){
+ Iterator itr = edgesToBeDeindexed.iterator();
+ while( itr.hasNext() ) {
+ batch.deindex( edge, ( CandidateResult ) itr.next());
+ }
+ return batch;
+ }
-
-
-
-
}
[06/12] incubator-usergrid git commit: Finishes testing of connections
Posted by gr...@apache.org.
Finishes testing of connections
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b59abac3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b59abac3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b59abac3
Branch: refs/heads/USERGRID-608
Commit: b59abac30fbdb001ef2d2bfbc4e7952e226ebc51
Parents: aa9153a
Author: Todd Nine <tn...@apigee.com>
Authored: Thu May 21 17:28:04 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu May 21 17:28:04 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 32 ++++------
.../pipeline/builder/IdBuilder.java | 5 ++
.../pipeline/read/collect/IdResumeFilter.java | 61 ++++++++++++++++++++
.../results/ConnectionRefQueryExecutor.java | 2 +-
.../results/EntityQueryExecutor.java | 2 +-
.../results/ObservableQueryExecutor.java | 4 +-
6 files changed, 81 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b59abac3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index be143ce..c766a1b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -37,7 +37,6 @@ import org.apache.usergrid.corepersistence.pipeline.builder.PipelineBuilderFacto
import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.corepersistence.results.ConnectionRefQueryExecutor;
import org.apache.usergrid.corepersistence.results.EntityQueryExecutor;
-import org.apache.usergrid.corepersistence.results.ObservableQueryExecutor;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.ConnectedEntityRef;
@@ -268,7 +267,7 @@ public class CpRelationManager implements RelationManager {
GraphManager gm = managerCache.getGraphManager( applicationScope );
Observable<Edge> edges = gm.loadEdgeVersions(
new SimpleSearchByEdge( new SimpleId( headEntity.getUuid(), headEntity.getType() ), edgeType, entityId,
- Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ) );
+ Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent() ) );
return edges.toBlocking().firstOrDefault( null ) != null;
}
@@ -933,37 +932,28 @@ public class CpRelationManager implements RelationManager {
cpHeadEntity.getId() );
-
-
- if(query.getResultsLevel() == Level.REFS){
+ if ( query.getResultsLevel() == Level.REFS || query.getResultsLevel() == Level.IDS ) {
final IdBuilder traversedIds;
- if(query.isGraphSearch()){
-
- traversedIds = pipelineBuilder.traverseConnection( connection, entityType );
-
+ if ( query.isGraphSearch() ) {
+ traversedIds = pipelineBuilder.traverseConnection( connection, entityType );
}
- else
- {
- traversedIds = pipelineBuilder.searchConnection( connection, query.getQl().get(), entityType ).loadIds();
-
+ else {
+ traversedIds =
+ pipelineBuilder.searchConnection( connection, query.getQl().get(), entityType ).loadIds();
}
- final Observable<ResultsPage<ConnectionRef>> results = traversedIds.loadConnectionRefs(
- cpHeadEntity.getId(), connection ).build();
+ //create connection refs
- return new ConnectionRefQueryExecutor( results ).next();
+ final Observable<ResultsPage<ConnectionRef>> results =
+ traversedIds.loadConnectionRefs( cpHeadEntity.getId(), connection ).build();
+ return new ConnectionRefQueryExecutor( results ).next();
}
- if(query.getResultsLevel() == Level.IDS){
-
- throw new UnsupportedOperationException( "Not yet implemented" );
- }
-
//we want to load all entities
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b59abac3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
index 4291ea9..0f784a6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/builder/IdBuilder.java
@@ -24,8 +24,11 @@ import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
import org.apache.usergrid.corepersistence.pipeline.read.FilterFactory;
import org.apache.usergrid.corepersistence.pipeline.Pipeline;
import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.corepersistence.pipeline.read.ResultsPage;
import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefFilter;
import org.apache.usergrid.corepersistence.pipeline.read.collect.ConnectionRefResumeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.IdResumeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.collect.ResultsPageCollector;
import org.apache.usergrid.corepersistence.pipeline.read.search.Candidate;
import org.apache.usergrid.persistence.ConnectionRef;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -33,6 +36,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
+import rx.Observable;
+
/**
* A builder to transition from emitting Ids in the pipeline into other operations
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b59abac3/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/IdResumeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/IdResumeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/IdResumeFilter.java
new file mode 100644
index 0000000..e9fd8de
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/collect/IdResumeFilter.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.collect;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * A filter that is used when we can potentially serialize pages via cursor. This will filter the first result, only if
+ * it matches the Id that was set
+ */
+public class IdResumeFilter extends AbstractPathFilter<Id, Id, Id> {
+
+
+ @Override
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterResultObservable ) {
+
+ //filter only the first id, then map into our path for our next pass
+
+
+ //skip our first and emit if neccessary
+ return filterResultObservable.skipWhile( filterResult -> {
+
+ final Optional<Id> startFromCursor = getSeekValue();
+
+ return startFromCursor.isPresent() && startFromCursor.get().equals( filterResult.getValue() );
+ } );
+ }
+
+
+ @Override
+ protected CursorSerializer<Id> getCursorSerializer() {
+ return IdCursorSerializer.INSTANCE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b59abac3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java
index 798c9c7..3dfd98a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefQueryExecutor.java
@@ -36,7 +36,7 @@ import rx.Observable;
/**
- * Processes our results of entities and turns them into
+ * Processes our results of connection refs
*/
@Deprecated//Required for 1.0 compatibility
public class ConnectionRefQueryExecutor extends ObservableQueryExecutor<ConnectionRef> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b59abac3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java
index bc9001e..0e18e31 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityQueryExecutor.java
@@ -35,7 +35,7 @@ import rx.Observable;
/**
- * Processes our results of entities and turns them into
+ * Processes our results of entities
*/
@Deprecated//Required for 1.0 compatibility
public class EntityQueryExecutor extends ObservableQueryExecutor<Entity> {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b59abac3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
index ff44416..fce1fb2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
@@ -60,7 +60,7 @@ public abstract class ObservableQueryExecutor<T> implements QueryExecutor {
* @param resultsPage
* @return
*/
- protected abstract Results createResults( final ResultsPage resultsPage );
+ protected abstract Results createResults( final ResultsPage<T> resultsPage );
@@ -69,7 +69,7 @@ public abstract class ObservableQueryExecutor<T> implements QueryExecutor {
* @param resultsPage
* @return
*/
- private Results createResultsInternal( final ResultsPage resultsPage ) {
+ private Results createResultsInternal( final ResultsPage<T> resultsPage ) {
final Results results = createResults( resultsPage );
[08/12] incubator-usergrid git commit: Updated bi-directional to
create 2 directional indexes
Posted by gr...@apache.org.
Updated bi-directional to create 2 directional indexes
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f7b47a84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f7b47a84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f7b47a84
Branch: refs/heads/USERGRID-608
Commit: f7b47a845bcbe30c066d8d8855aa3e3572b0a548
Parents: 17d2d73
Author: Todd Nine <tn...@apigee.com>
Authored: Thu May 21 18:21:54 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu May 21 18:21:54 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 26 ++++++++---
.../corepersistence/index/IndexServiceImpl.java | 46 +-------------------
2 files changed, 20 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7b47a84/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 4a2d887..c31288a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -72,6 +72,7 @@ import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.schema.CollectionInfo;
+import org.apache.usergrid.utils.InflectionUtils;
import org.apache.usergrid.utils.MapUtils;
import com.google.common.base.Optional;
@@ -397,16 +398,27 @@ public class CpRelationManager implements RelationManager {
logger.debug( "Wrote edge {}", edge );
}
- indexService.queueNewEdge(applicationScope, memberEntity, edge);
- //reverse
- if(!cpHeadEntity.getId().getType().equals("application")) {
- String pluralType = InflectionUtils.pluralize(cpHeadEntity.getId().getType());
- final Edge reverseEdge = createCollectionEdge(memberEntity.getId(), pluralType, cpHeadEntity.getId());
- gm.writeEdge(reverseEdge).toBlocking().last();
- indexService.queueNewEdge(applicationScope, cpHeadEntity, reverseEdge);
+
+ //check if we need to reverse our edges
+
+
+ final String linkedCollection = collection.getLinkedCollection();
+
+ /**
+ * Nothing to link
+ */
+ if ( linkedCollection != null ) {
+ String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
+ final Edge reverseEdge = createCollectionEdge( memberEntity.getId(), pluralType, cpHeadEntity.getId() );
+ gm.writeEdge( reverseEdge ).toBlocking().last();
}
+ indexService.queueNewEdge( applicationScope, memberEntity, edge );
+
+
+
+
if ( logger.isDebugEnabled() ) {
logger.debug( "Added entity {}:{} to collection {}", new Object[] {
itemRef.getUuid().toString(), itemRef.getType(), collectionName
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7b47a84/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 9bb0728..b57ea92 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -102,20 +102,13 @@ public class IndexServiceImpl implements IndexService {
//we may have to index we're indexing from source->target here
final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeFromSource( edge ) );
-
- //we might or might not need to index from target-> source
- final Observable<IndexEdge> edgesToSource = getIndexEdgesAsTarget( gm, entityId );
-
-
- //merge the edges together
- final Observable<IndexEdge> observable = Observable.merge( sourceEdgesToIndex, edgesToSource);
//do our observable for batching
//try to send a whole batch if we can
//do our observable for batching
//try to send a whole batch if we can
- final Observable<IndexOperationMessage> batches = observable.buffer( indexFig.getIndexBatchSize() )
+ final Observable<IndexOperationMessage> batches = sourceEdgesToIndex.buffer( indexFig.getIndexBatchSize() )
//map into batches based on our buffer size
.flatMap( buffer -> Observable.from( buffer )
@@ -182,44 +175,7 @@ public class IndexServiceImpl implements IndexService {
}
- /**
- * Get index edges to the target. Used in only certain entity types, such as roles, users, groups etc
- * where we doubly index on both directions of the edge
- *
- * @param graphManager The graph manager
- * @param entityId The entity's id
- */
- private Observable<IndexEdge> getIndexEdgesAsTarget( final GraphManager graphManager, final Id entityId ) {
-
- final String collectionName = InflectionUtils.pluralize( entityId.getType() );
-
-
- final CollectionInfo collection = getDefaultSchema().getCollection( Application.ENTITY_TYPE, collectionName );
- //nothing to do
- if ( collection == null ) {
- return Observable.empty();
- }
-
-
- final String linkedCollection = collection.getLinkedCollection();
-
- /**
- * Nothing to link
- */
- if ( linkedCollection == null ) {
- return Observable.empty();
- }
-
-
- /**
- * An observable of sizes as we execute batches
- *
- * we're indexing from target->source here
- */
- return edgesObservable.getEdgesFromSource( graphManager, entityId, linkedCollection )
- .map( edge -> generateScopeFromTarget( edge ) );
- }
[07/12] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
USERGRID-641
Posted by gr...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-641
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/17d2d730
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/17d2d730
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/17d2d730
Branch: refs/heads/USERGRID-608
Commit: 17d2d7303a321e4b61a8508ea2a789f7b77c68e0
Parents: b59abac 53563e8
Author: Todd Nine <tn...@apigee.com>
Authored: Thu May 21 18:21:01 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu May 21 18:21:01 2015 -0600
----------------------------------------------------------------------
portal/config.js | 4 +-
portal/js/global/ug-service.js | 4 +-
portal/js/users/users-controller.js | 19 +-
.../usergrid/corepersistence/CoreModule.java | 4 +
.../corepersistence/CpEntityManager.java | 2 +-
.../corepersistence/CpEntityManagerFactory.java | 14 -
.../corepersistence/CpRelationManager.java | 54 +--
.../asyncevents/EventBuilder.java | 6 +-
.../asyncevents/EventBuilderImpl.java | 18 +-
.../asyncevents/InMemoryAsyncEventService.java | 9 +-
.../asyncevents/SQSAsyncEventService.java | 9 +-
.../index/EdgeScopeSerializer.java | 41 +++
.../index/EntityIndexOperation.java | 46 +++
.../index/IndexProcessorFig.java | 6 +-
.../corepersistence/index/IndexServiceImpl.java | 3 +-
.../corepersistence/index/ReIndexAction.java | 6 +-
.../index/ReIndexRequestBuilder.java | 86 +++++
.../index/ReIndexRequestBuilderImpl.java | 122 +++++++
.../corepersistence/index/ReIndexService.java | 89 +++--
.../index/ReIndexServiceImpl.java | 251 ++++++++++++--
.../cursor/AbstractCursorSerializer.java | 2 +-
.../pipeline/cursor/CursorSerializerUtil.java | 54 ++-
.../pipeline/cursor/RequestCursor.java | 9 +-
.../pipeline/cursor/ResponseCursor.java | 49 +--
.../pipeline/read/AbstractPathFilter.java | 30 --
.../pipeline/read/CursorSeek.java | 53 +++
.../rx/impl/AllEntityIdsObservable.java | 5 +-
.../rx/impl/AllEntityIdsObservableImpl.java | 6 +-
.../corepersistence/util/CpNamingUtils.java | 3 +-
.../util/SerializableMapper.java | 91 -----
.../persistence/EntityManagerFactory.java | 2 -
.../usergrid/persistence/RelationManager.java | 2 -
.../rx/EdgesToTargetObservableIT.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 346 ++++++++-----------
.../data/MigrationInfoSerializationImpl.java | 4 +-
.../migration/schema/MigrationManagerImpl.java | 14 +-
.../core/migration/util/AstayanxUtils.java | 49 +++
.../graph/serialization/EdgesObservable.java | 24 +-
.../serialization/impl/EdgesObservableImpl.java | 12 +-
.../impl/TargetIdObservableImpl.java | 2 +-
.../impl/migration/EdgeDataMigrationImpl.java | 2 +-
.../persistence/index/impl/IndexingUtils.java | 2 +
stack/pom.xml | 13 +
.../org/apache/usergrid/rest/IndexResource.java | 342 +++++++++---------
.../main/resources/usergrid-rest-context.xml | 3 -
.../resources/usergrid-rest-deploy-context.xml | 1 -
.../usergrid/management/OrganizationIT.java | 8 +-
.../resources/usergrid-custom-test.properties | 1 +
48 files changed, 1189 insertions(+), 737 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/17d2d730/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/17d2d730/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/17d2d730/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c766a1b,e6ab3ba..4a2d887
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -656,20 -634,16 +637,16 @@@ public class CpRelationManager implemen
boolean found;
long current = System.currentTimeMillis(), length = 0;
do {
- results = searchCollection(collectionName, query);
+ results = searchCollection( collectionName, query );
length = System.currentTimeMillis() - current;
found = expectedResults == results.size();
- if(found){
+ if ( found ) {
break;
}
- Thread.sleep( sleepTime );
- }
- while ( !found && length <= maxLength );
- if ( logger.isInfoEnabled() ) {
- logger.info( String
- .format( "Consistent Search finished in %s, results=%s, expected=%s...dumping stack", length,
- results.size(), expectedResults ) );
- Thread.dumpStack();
+ Thread.sleep(sleepTime);
+ }while (!found && length <= maxLength);
+ if(logger.isInfoEnabled()){
+ logger.info(String.format("Consistent Search finished in %s, results=%s, expected=%s...dumping stack",length, results.size(),expectedResults));
}
return results;
}
@@@ -970,8 -924,9 +947,9 @@@
}
- final Observable<ResultsPage> resultsObservable = readPipelineBuilder.execute();
- return new ObservableQueryExecutor( resultsObservable ).next();
++
+ return new EntityQueryExecutor( results ).next();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/17d2d730/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractPathFilter.java
----------------------------------------------------------------------
[02/12] incubator-usergrid git commit: Refactors operations into
easier build pattern. Pipeline still need some work.
Posted by gr...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
new file mode 100644
index 0000000..d0b6af9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -0,0 +1,232 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.search;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+
+/**
+ * Loads entities from an incoming CandidateResult emissions into entities, then streams them on
+ * performs internal buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries. It is up to the
+ * collector to determine when to stop streaming entities.
+ */
+public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate>, FilterResult<Entity>> {
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final EntityIndexFactory entityIndexFactory;
+
+
+ @Inject
+ public CandidateEntityFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final EntityIndexFactory entityIndexFactory ) {
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.entityIndexFactory = entityIndexFactory;
+ }
+
+
+ @Override
+ public Observable<FilterResult<Entity>> call(
+ final Observable<FilterResult<Candidate>> candidateResultsObservable ) {
+
+
+ /**
+ * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
+ * objects
+ */
+
+ final ApplicationScope applicationScope = pipelineContext.getApplicationScope();
+
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+
+ final ApplicationEntityIndex applicationIndex =
+ entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
+ //buffer them to get a page size we can make 1 network hop
+ final Observable<FilterResult<Entity>> searchIdSetObservable = candidateResultsObservable.buffer( pipelineContext.getLimit() )
+
+ //load them
+ .flatMap( candidateResults -> {
+ //flatten toa list of ids to load
+ final Observable<List<Id>> candidateIds =
+ Observable.from( candidateResults ).map( filterResultCandidate -> filterResultCandidate.getValue().getCandidateResult().getId() ).toList();
+
+ //load the ids
+ final Observable<EntitySet> entitySetObservable =
+ candidateIds.flatMap( ids -> entityCollectionManager.load( ids ) );
+
+ //now we have a collection, validate our canidate set is correct.
+
+ return entitySetObservable.map(
+ entitySet -> new EntityVerifier( applicationIndex.createBatch(), entitySet,
+ candidateResults ) ).doOnNext( entityCollector -> entityCollector.merge() )
+ .flatMap(
+ entityCollector -> Observable.from( entityCollector.getResults() ) );
+ } );
+
+ //if we filter all our results, we want to continue to try the next page
+ return searchIdSetObservable;
+ }
+
+
+
+
+ /**
+ * Our collector to collect entities. Not quite a true collector, but works within our operational flow as this state is mutable and difficult to represent functionally
+ */
+ private static final class EntityVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
+ private List<FilterResult<Entity>> results = new ArrayList<>();
+
+ private final EntityIndexBatch batch;
+ private final List<FilterResult<Candidate>> candidateResults;
+ private final EntitySet entitySet;
+
+
+ public EntityVerifier( final EntityIndexBatch batch, final EntitySet entitySet,
+ final List<FilterResult<Candidate>> candidateResults ) {
+ this.batch = batch;
+ this.entitySet = entitySet;
+ this.candidateResults = candidateResults;
+ this.results = new ArrayList<>( entitySet.size() );
+ }
+
+
+ /**
+ * Merge our candidates and our entity set into results
+ */
+ public void merge() {
+
+ for ( final FilterResult<Candidate> candidateResult : candidateResults ) {
+ validate( candidateResult );
+ }
+
+ batch.execute();
+ }
+
+
+ public List<FilterResult<Entity>> getResults() {
+ return results;
+ }
+
+
+ public EntityIndexBatch getBatch() {
+ return batch;
+ }
+
+
+ private void validate( final FilterResult<Candidate> filterResult ) {
+
+ final Candidate candidate = filterResult.getValue();
+ final CandidateResult candidateResult = candidate.getCandidateResult();
+ final SearchEdge searchEdge = candidate.getSearchEdge();
+ final Id candidateId = candidateResult.getId();
+ final UUID candidateVersion = candidateResult.getVersion();
+
+
+ final MvccEntity entity = entitySet.getEntity( candidateId );
+
+
+ //doesn't exist warn and drop
+ if ( entity == null ) {
+ logger.warn(
+ "Searched and received candidate with entityId {} and version {}, yet was not found in cassandra."
+ + " Ignoring since this could be a region sync issue",
+ candidateId, candidateVersion );
+
+
+ //TODO trigger an audit after a fail count where we explicitly try to repair from other regions
+
+ return;
+
+ }
+
+
+ final UUID entityVersion = entity.getVersion();
+ final Id entityId = entity.getId();
+
+
+
+
+
+ //entity is newer than ES version, could be an update or the entity is marked as deleted
+ if ( UUIDComparator.staticCompare( entityVersion, candidateVersion ) > 0 || !entity.getEntity().isPresent()) {
+
+ logger.warn( "Deindexing stale entity on edge {} for entityId {} and version {}",
+ new Object[] { searchEdge, entityId, entityVersion } );
+ batch.deindex( searchEdge, entityId, candidateVersion );
+ return;
+ }
+
+ //ES is newer than cass, it means we haven't repaired the record in Cass, we don't want to
+ //remove the ES record, since the read in cass should cause a read repair, just ignore
+ if ( UUIDComparator.staticCompare( candidateVersion, entityVersion ) > 0 ) {
+
+ logger.warn(
+ "Found a newer version in ES over cassandra for edge {} for entityId {} and version {}. Repair "
+ + "should be run", new Object[] { searchEdge, entityId, entityVersion } );
+
+ //TODO trigger an audit after a fail count where we explicitly try to repair from other regions
+
+ return;
+ }
+
+ //they're the same add it
+
+ final Entity returnEntity = entity.getEntity().get();
+
+ final Optional<EdgePath> parent = filterResult.getPath();
+
+ final FilterResult<Entity> toReturn = new FilterResult<>( returnEntity, parent );
+
+ results.add( toReturn );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
new file mode 100644
index 0000000..1ef358a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.search;
+
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+
+/**
+ * Responsible for verifying candidate result versions, then emitting the Ids of these versions Input is a batch of
+ * candidate results, output is a stream of validated Ids
+ */
+public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, FilterResult<Id>> {
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final EntityIndexFactory entityIndexFactory;
+
+
+ @Inject
+ public CandidateIdFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final EntityIndexFactory entityIndexFactory ) {
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.entityIndexFactory = entityIndexFactory;
+ }
+
+
+ @Override
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Candidate>> filterResultObservable ) {
+
+
+ /**
+ * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
+ * objects
+ */
+
+ final ApplicationScope applicationScope = pipelineContext.getApplicationScope();
+
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+
+ final ApplicationEntityIndex applicationIndex =
+ entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
+ final Observable<FilterResult<Id>> searchIdSetObservable =
+ filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( candidateResults -> {
+ //flatten toa list of ids to load
+ final Observable<List<Id>> candidateIds = Observable.from( candidateResults ).map(
+ candidate -> candidate.getValue().getCandidateResult().getId() ).toList();
+
+ //load the ids
+ final Observable<VersionSet> versionSetObservable =
+ candidateIds.flatMap( ids -> entityCollectionManager.getLatestVersion( ids ) );
+
+ //now we have a collection, validate our canidate set is correct.
+
+ return versionSetObservable.map(
+ entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet,
+ candidateResults ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
+ entityCollector -> Observable.from( entityCollector.collectResults() ) );
+ } );
+
+ return searchIdSetObservable;
+ }
+
+
+ /**
+ * Map a new cp entity to an old entity. May be null if not present
+ */
+ private static final class EntityCollector {
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityCollector.class );
+ private List<FilterResult<Id>> results = new ArrayList<>();
+
+ private final EntityIndexBatch batch;
+ private final List<FilterResult<Candidate>> candidateResults;
+ private final VersionSet versionSet;
+
+
+ public EntityCollector( final EntityIndexBatch batch, final VersionSet versionSet,
+ final List<FilterResult<Candidate>> candidateResults ) {
+ this.batch = batch;
+ this.versionSet = versionSet;
+ this.candidateResults = candidateResults;
+ this.results = new ArrayList<>( versionSet.size() );
+ }
+
+
+ /**
+ * Merge our candidates and our entity set into results
+ */
+ public void merge() {
+
+ for ( final FilterResult<Candidate> candidateResult : candidateResults ) {
+ validate( candidateResult );
+ }
+
+ batch.execute();
+ }
+
+
+ public List<FilterResult<Id>> collectResults() {
+ return results;
+ }
+
+
+ /**
+ * Validate each candidate results vs the data loaded from cass
+ */
+ private void validate( final FilterResult<Candidate> filterCandidate ) {
+
+ final CandidateResult candidateResult = filterCandidate.getValue().getCandidateResult();
+
+ final SearchEdge searchEdge = filterCandidate.getValue().getSearchEdge();
+
+ final MvccLogEntry logEntry = versionSet.getMaxVersion( candidateResult.getId() );
+
+ final UUID candidateVersion = candidateResult.getVersion();
+
+ final UUID entityVersion = logEntry.getVersion();
+
+ final Id entityId = logEntry.getEntityId();
+
+ //entity is newer than ES version
+ if ( UUIDComparator.staticCompare( entityVersion, candidateVersion ) > 0 ) {
+
+ logger.warn( "Deindexing stale entity on edge {} for entityId {} and version {}",
+ new Object[] { searchEdge, entityId, entityVersion } );
+ batch.deindex( searchEdge, entityId, entityVersion );
+ return;
+ }
+
+ //ES is newer than cass, it means we haven't repaired the record in Cass, we don't want to
+ //remove the ES record, since the read in cass should cause a read repair, just ignore
+ if ( UUIDComparator.staticCompare( candidateVersion, entityVersion ) > 0 ) {
+
+ logger.warn(
+ "Found a newer version in ES over cassandra for edge {} for entityId {} and version {}. Repair "
+ + "should be run", new Object[] { searchEdge, entityId, entityVersion } );
+ }
+
+ //they're the same add it
+
+ final FilterResult<Id> result = new FilterResult<>( entityId, filterCandidate.getPath() );
+
+ results.add( result );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/ElasticsearchCursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/ElasticsearchCursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/ElasticsearchCursorSerializer.java
new file mode 100644
index 0000000..7cf5a78
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/ElasticsearchCursorSerializer.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.search;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.AbstractCursorSerializer;
+
+
+/**
+ * ElasticSearch cursor serializer
+ */
+public class ElasticsearchCursorSerializer extends AbstractCursorSerializer<Integer> {
+
+
+ public static final ElasticsearchCursorSerializer INSTANCE = new ElasticsearchCursorSerializer();
+
+ @Override
+ protected Class<Integer> getType() {
+ return Integer.class;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Elasticsearchdiagram.jpg
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Elasticsearchdiagram.jpg b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Elasticsearchdiagram.jpg
new file mode 100644
index 0000000..08970e3
Binary files /dev/null and b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/Elasticsearchdiagram.jpg differ
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java
new file mode 100644
index 0000000..a6edd56
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchCollectionFilter.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.search;
+
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionSearchEdge;
+
+
+public class SearchCollectionFilter extends AbstractElasticSearchFilter {
+
+ private final String collectionName;
+ private final String entityType;
+
+ /**
+ * Create a new instance of our command
+ *
+ * @param entityIndexFactory The entity index factory used to search
+ * @param metricsFactory The metrics factory for metrics
+ * @param collectionName The name of the collection
+ * @param entityType The entity type
+ */
+ @Inject
+ public SearchCollectionFilter( final EntityIndexFactory entityIndexFactory, final MetricsFactory metricsFactory,
+ @Assisted( "query" ) final String query,
+ @Assisted( "collectionName" ) final String collectionName,
+ @Assisted( "entityType" ) final String entityType ) {
+ super( entityIndexFactory, metricsFactory, query );
+ this.collectionName = collectionName;
+ this.entityType = entityType;
+ }
+
+
+
+ @Override
+ protected SearchTypes getSearchTypes() {
+ final SearchTypes types = SearchTypes.fromTypes( entityType );
+
+ return types;
+ }
+
+
+ @Override
+ protected SearchEdge getSearchEdge( final Id incomingId ) {
+ final SearchEdge searchEdge = createCollectionSearchEdge( incomingId, collectionName );
+
+ return searchEdge;
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java
new file mode 100644
index 0000000..82d7377
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/SearchConnectionFilter.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.search;
+
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchEdge;
+
+
+public class SearchConnectionFilter extends AbstractElasticSearchFilter {
+
+
+ private final String connectionName;
+ private final Optional<String> connectedEntityType;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ @Inject
+ public SearchConnectionFilter( final EntityIndexFactory entityIndexFactory, final MetricsFactory metricsFactory,
+ @Assisted( "query" ) final String query,
+ @Assisted( "connectionName" ) final String connectionName,
+ @Assisted( "connectedEntityType" ) final Optional<String> connectedEntityType ) {
+ super( entityIndexFactory, metricsFactory, query );
+
+ this.connectionName = connectionName;
+ this.connectedEntityType = connectedEntityType;
+ }
+
+
+ @Override
+ protected SearchTypes getSearchTypes() {
+ final SearchTypes searchTypes = SearchTypes.fromNullableTypes( connectedEntityType.orNull() );
+
+ return searchTypes;
+ }
+
+
+ @Override
+ protected SearchEdge getSearchEdge( final Id id ) {
+ final SearchEdge searchEdge = createConnectionSearchEdge( id, connectionName );
+
+ return searchEdge;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphEdgeByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphEdgeByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphEdgeByIdFilter.java
new file mode 100644
index 0000000..5b3a42e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphEdgeByIdFilter.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.pipeline.PipelineOperation;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+
+/**
+ * Filter should take and Id and a graph edge, and ensure the connection between the two exists
+ */
+public abstract class AbstractReadGraphEdgeByIdFilter extends AbstractFilter<FilterResult<Id>, FilterResult<Id>> implements
+ PipelineOperation<FilterResult<Id>, FilterResult<Id>> {
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final Id targetId;
+
+
+ @Inject
+ public AbstractReadGraphEdgeByIdFilter( final GraphManagerFactory graphManagerFactory, @Assisted final Id
+ targetId ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.targetId = targetId;
+ }
+
+
+ @Override
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterValueObservable ) {
+
+ final GraphManager gm = graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() );
+
+ return filterValueObservable.flatMap( filterValue -> {
+ final String edgeTypeName = getEdgeName();
+ final Id id = filterValue.getValue();
+
+ //create our search
+ final SearchByEdge searchByEdge =
+ new SimpleSearchByEdge( id, edgeTypeName, targetId, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ Optional.absent() );
+
+ //load the versions of the edge, take the first since that's all we need to validate existence, then emit the target node
+ return gm.loadEdgeVersions( searchByEdge ).take( 1 ).map( edge -> edge.getTargetNode() ).map( targetId -> new FilterResult<>(targetId, filterValue.getPath()));
+ } );
+ }
+
+
+ /**
+ * Get the name of the edge to be used in the seek
+ */
+ protected abstract String getEdgeName();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
new file mode 100644
index 0000000..f477092
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/AbstractReadGraphFilter.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * Command for reading graph edges
+ */
+public abstract class AbstractReadGraphFilter extends AbstractPathFilter<Id, Id, Edge> {
+
+ private final GraphManagerFactory graphManagerFactory;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory ) {
+ this.graphManagerFactory = graphManagerFactory;
+ }
+
+
+ @Override
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> previousIds ) {
+
+
+ //get the graph manager
+ final GraphManager graphManager =
+ graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() );
+
+
+ final String edgeName = getEdgeTypeName();
+ final EdgeState edgeCursorState = new EdgeState();
+
+
+ //return all ids that are emitted from this edge
+ return previousIds.flatMap( previousFilterValue -> {
+
+ //set our our constant state
+ final Optional<Edge> startFromCursor = getSeekValue();
+ final Id id = previousFilterValue.getValue();
+
+
+ final SimpleSearchByEdgeType search =
+ new SimpleSearchByEdgeType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ startFromCursor );
+
+ /**
+ * TODO, pass a message with pointers to our cursor values to be generated later
+ */
+ return graphManager.loadEdgesFromSource( search )
+ //set the edge state for cursors
+ .doOnNext( edge -> edgeCursorState.update( edge ) )
+
+ //map our id from the target edge and set our cursor every edge we traverse
+ .map( edge -> createFilterResult( edge.getTargetNode(), edgeCursorState.getCursorEdge(),
+ previousFilterValue.getPath() ) );
+ } );
+ }
+
+
+ @Override
+ protected FilterResult<Id> createFilterResult( final Id emit, final Edge cursorValue,
+ final Optional<EdgePath> parent ) {
+
+ //if it's our first pass, there's no cursor to generate
+ if(cursorValue == null){
+ return new FilterResult<>( emit, parent );
+ }
+
+ return super.createFilterResult( emit, cursorValue, parent );
+ }
+
+
+ @Override
+ protected CursorSerializer<Edge> getCursorSerializer() {
+ return EdgeCursorSerializer.INSTANCE;
+ }
+
+
+ /**
+ * Get the edge type name we should use when traversing
+ */
+ protected abstract String getEdgeTypeName();
+
+
+ /**
+ * Wrapper class. Because edges seek > the last returned, we need to keep our n-1 value. This will be our cursor We
+ * always try to seek to the same position as we ended. Since we don't deal with a persistent read result, if we
+ * seek to a value = to our last, we may skip data.
+ */
+ private final class EdgeState {
+
+ private Edge cursorEdge = null;
+ private Edge currentEdge = null;
+
+
+ /**
+ * Update the pointers
+ */
+ private void update( final Edge newEdge ) {
+ cursorEdge = currentEdge;
+ currentEdge = newEdge;
+ }
+
+
+ /**
+ * Get the edge to use in cursors for resume
+ */
+ private Edge getCursorEdge() {
+ return cursorEdge;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java
new file mode 100644
index 0000000..8d9bf6f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EdgeCursorSerializer.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.AbstractCursorSerializer;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
+
+
+/**
+ * Edge cursor serializer
+ */
+public class EdgeCursorSerializer extends AbstractCursorSerializer<Edge> {
+
+
+ public static final EdgeCursorSerializer INSTANCE = new EdgeCursorSerializer();
+
+ @Override
+ protected Class<SimpleEdge> getType() {
+ return SimpleEdge.class;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityIdFilter.java
new file mode 100644
index 0000000..003038a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityIdFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+
+/**
+ * This command is a stopgap to make migrating 1.0 code easier. Once full traversal has been implemented, this should
+ * be removed
+ */
+public class EntityIdFilter extends AbstractFilter<FilterResult<Id>, FilterResult<Id>>{
+
+ private final Id entityId;
+
+
+ @Inject
+ public EntityIdFilter( @Assisted final Id entityId ) {this.entityId = entityId;}
+
+
+
+ @Override
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterValueObservable ) {
+ //ignore what our input was, and simply emit the id specified
+ return filterValueObservable.map( idFilterResult -> new FilterResult( entityId, idFilterResult.getPath() ));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java
new file mode 100644
index 0000000..41507e9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/EntityLoadVerifyFilter.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+
+/**
+ * Loads entities from a set of Ids. and verify they are valid
+ *
+ * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
+ */
+public class EntityLoadVerifyFilter extends AbstractFilter<FilterResult<Id>, FilterResult<Entity>>{
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public EntityLoadVerifyFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public Observable<FilterResult<Entity>> call( final Observable<FilterResult<Id>> filterResultObservable ) {
+
+
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( pipelineContext.getApplicationScope() );
+
+ //it's more efficient to make 1 network hop to get everything, then drop our results if required
+ final Observable<FilterResult<Entity>> entityObservable =
+ filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( bufferedIds -> {
+
+ final Observable<EntitySet> entitySetObservable =
+ Observable.from( bufferedIds ).map( filterResultId -> filterResultId.getValue() ).toList()
+ .flatMap( ids -> entityCollectionManager.load( ids ) );
+
+
+ //now we have a collection, validate our canidate set is correct.
+
+ return entitySetObservable.map( entitySet -> new EntityVerifier( entitySet, bufferedIds ) )
+ .doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
+ entityCollector -> Observable.from( entityCollector.getResults() ) );
+ } );
+
+ return entityObservable;
+ }
+
+
+ /**
+ * Our collector to collect entities. Not quite a true collector, but works within our operational flow as this
+ * state is mutable and difficult to represent functionally
+ */
+ private static final class EntityVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
+ private List<FilterResult<Entity>> results = new ArrayList<>();
+
+ private final List<FilterResult<Id>> candidateResults;
+ private final EntitySet entitySet;
+
+
+ public EntityVerifier( final EntitySet entitySet, final List<FilterResult<Id>> candidateResults ) {
+ this.entitySet = entitySet;
+ this.candidateResults = candidateResults;
+ this.results = new ArrayList<>( entitySet.size() );
+ }
+
+
+ /**
+ * Merge our candidates and our entity set into results
+ */
+ public void merge() {
+
+ for ( final FilterResult<Id> candidateResult : candidateResults ) {
+ validate( candidateResult );
+ }
+ }
+
+
+ public List<FilterResult<Entity>> getResults() {
+ return results;
+ }
+
+
+ private void validate( final FilterResult<Id> filterResult ) {
+
+ final Id candidateId = filterResult.getValue();
+
+
+ final MvccEntity entity = entitySet.getEntity( candidateId );
+
+
+ //doesn't exist warn and drop
+ if ( entity == null || !entity.getEntity().isPresent() ) {
+ logger.warn( "Read graph edge and received candidate with entityId {}, yet was not found in cassandra."
+ + " Ignoring since this could be a region sync issue", candidateId );
+
+
+ //TODO trigger an audit after a fail count where we explicitly try to repair from other regions
+
+ return;
+ }
+
+ //it exists, add it
+
+ final Entity returnEntity = entity.getEntity().get();
+
+ final Optional<EdgePath> parent = filterResult.getPath();
+
+ final FilterResult<Entity> toReturn = new FilterResult<>( returnEntity, parent );
+
+ results.add( toReturn );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/GraphDiagram.jpg
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/GraphDiagram.jpg b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/GraphDiagram.jpg
new file mode 100644
index 0000000..c0308bd
Binary files /dev/null and b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/GraphDiagram.jpg differ
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionByIdFilter.java
new file mode 100644
index 0000000..71d606c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionByIdFilter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+
+/**
+ * Read an edge in the graph to verify it's existence by id
+ */
+public class ReadGraphCollectionByIdFilter extends AbstractReadGraphEdgeByIdFilter{
+
+ private final String collectionName;
+
+ @Inject
+ public ReadGraphCollectionByIdFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName, @Assisted final Id targetId ) {
+ super( graphManagerFactory, targetId );
+ this.collectionName = collectionName;
+ }
+
+
+ @Override
+ protected String getEdgeName() {
+ return CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
new file mode 100644
index 0000000..dc39f5c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphCollectionFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromCollectionName;
+
+
+/**
+ * Command for reading graph edges on a collection
+ */
+public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
+
+ private final String collectionName;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ @Inject
+ public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName ) {
+ super( graphManagerFactory );
+ this.collectionName = collectionName;
+ }
+
+
+ @Override
+ protected String getEdgeTypeName() {
+ return getEdgeTypeFromCollectionName( collectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByIdFilter.java
new file mode 100644
index 0000000..4f71202
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByIdFilter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+
+/**
+ * Read an edge in the graph to verify it's existence by id
+ */
+public class ReadGraphConnectionByIdFilter extends AbstractReadGraphEdgeByIdFilter{
+
+ private final String connectionName;
+
+ @Inject
+ public ReadGraphConnectionByIdFilter( final GraphManagerFactory graphManagerFactory,
+ @Assisted final String connectionName, @Assisted final Id targetId ) {
+ super( graphManagerFactory, targetId );
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected String getEdgeName() {
+ return CpNamingUtils.getEdgeTypeFromConnectionType( connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java
new file mode 100644
index 0000000..61ba4ad
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionByTypeFilter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.CursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractPathFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.FilterResult;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromConnectionType;
+
+
+/**
+ * Command for reading graph edges on a connection
+ */
+public class ReadGraphConnectionByTypeFilter extends AbstractPathFilter<Id, Id, Edge>{
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final String connectionName;
+ private final String entityType;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ @Inject
+ public ReadGraphConnectionByTypeFilter( final GraphManagerFactory graphManagerFactory,
+ @Assisted("connectionName") final String connectionName, @Assisted("entityType") final String entityType ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.connectionName = connectionName;
+ this.entityType = entityType;
+ }
+
+
+
+ @Override
+ public Observable<FilterResult<Id>> call( final Observable<FilterResult<Id>> filterResultObservable ) {
+
+ //get the graph manager
+ final GraphManager graphManager = graphManagerFactory.createEdgeManager( pipelineContext.getApplicationScope() );
+
+
+
+ final String edgeName = getEdgeTypeFromConnectionType( connectionName );
+
+
+ //return all ids that are emitted from this edge
+ return filterResultObservable.flatMap( idFilterResult -> {
+
+ //set our our constant state
+ final Optional<Edge> startFromCursor = getSeekValue();
+ final Id id = idFilterResult.getValue();
+
+ final SimpleSearchByIdType search =
+ new SimpleSearchByIdType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ entityType, startFromCursor );
+
+ return graphManager.loadEdgesFromSourceByType( search ).map(
+ edge -> createFilterResult( edge.getTargetNode(), edge, idFilterResult.getPath() ));
+ } );
+ }
+
+
+ @Override
+ protected CursorSerializer<Edge> getCursorSerializer() {
+ return EdgeCursorSerializer.INSTANCE;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
new file mode 100644
index 0000000..11ec5f8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/traverse/ReadGraphConnectionFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.traverse;
+
+
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromConnectionType;
+
+
+/**
+ * Command for reading graph edges on a connection
+ */
+public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
+
+ private final String connectionName;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ @Inject
+ public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String connectionName ) {
+ super( graphManagerFactory );
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected String getEdgeTypeName() {
+ return getEdgeTypeFromConnectionType( connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
index 0260d1d..c779bb7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
@@ -49,7 +49,7 @@ public class ObservableQueryExecutor implements QueryExecutor {
public Iterator<Results> iterator;
- public ObservableQueryExecutor( final Observable<ResultsPage> resultsObservable) {
+ public ObservableQueryExecutor( final Observable<ResultsPage<Entity>> resultsObservable) {
//map to our old results objects, return a default empty if required
this.resultsObservable = resultsObservable.map( resultsPage -> createResults( resultsPage ) ).defaultIfEmpty( new Results() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6d54dffc/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
index fd65ebf..7128dcf 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorTest.java
@@ -25,8 +25,8 @@ package org.apache.usergrid.corepersistence.pipeline.cursor;
import org.junit.Test;
import org.apache.usergrid.corepersistence.pipeline.read.EdgePath;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.ElasticsearchCursorSerializer;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.EdgeCursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.search.ElasticsearchCursorSerializer;
+import org.apache.usergrid.corepersistence.pipeline.read.traverse.EdgeCursorSerializer;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
[09/12] incubator-usergrid git commit: add is application method
Posted by gr...@apache.org.
add is application method
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5bb9aff4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5bb9aff4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5bb9aff4
Branch: refs/heads/USERGRID-608
Commit: 5bb9aff4bd399a62db51183d4e48d24846705f8a
Parents: 53563e8
Author: Shawn Feldman <sf...@apache.org>
Authored: Tue May 26 10:55:10 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Tue May 26 10:55:10 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/util/CpNamingUtils.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5bb9aff4/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index f4a4888..31dabea 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.corepersistence.util;
import java.util.UUID;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
@@ -270,4 +271,8 @@ public class CpNamingUtils {
}
+ public static boolean isApplication(Id id) {
+ Preconditions.checkNotNull(id);
+ return id.getType().equals("application");
+ }
}