You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/27 19:39:10 UTC
[01/20] incubator-usergrid git commit: Added elastic search timeout
to search calls.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-dev bd3d93959 -> ff2b3f142
Added elastic search timeout to search calls.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8b97a6db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8b97a6db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8b97a6db
Branch: refs/heads/two-dot-o-dev
Commit: 8b97a6dbff5914b94375e2ebf930ba7e14e71631
Parents: 2d1c8b8
Author: GERey <gr...@apigee.com>
Authored: Fri Apr 17 14:43:51 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Apr 17 14:43:51 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/persistence/index/IndexFig.java | 11 +++++++++++
.../persistence/index/impl/EsEntityIndexImpl.java | 2 +-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8b97a6db/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index c7be79d..2539696 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -100,6 +100,13 @@ public interface IndexFig extends GuicyFig {
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+ /**
+ * Timeout calls to elasticsearch.
+ * @return
+ */
+ public static final String ELASTICSEARCH_TIMEOUT = "elasticsearch.timeout";
+
+
@Default( "127.0.0.1" )
@Key( ELASTICSEARCH_HOSTS )
String getHosts();
@@ -209,4 +216,8 @@ public interface IndexFig extends GuicyFig {
@Default( "1000" )
@Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
long getQueueOfferTimeout();
+
+ @Default( "5000" )
+ @Key( ELASTICSEARCH_TIMEOUT )
+ long getElasticsearchTimeout();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8b97a6db/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 1838eec..9d8d278 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -503,7 +503,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
Timer.Context timeSearchCursor = cursorTimer.time();
- searchResponse = ssrb.execute().actionGet();
+ searchResponse = ssrb.execute().actionGet(config.getElasticsearchTimeout());
timeSearchCursor.stop();
}
catch ( Throwable t ) {
[06/20] incubator-usergrid git commit: Initial setup of command
pattern and wiring into legacy code
Posted by to...@apache.org.
Initial setup of command pattern and wiring into legacy code
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6098d9b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6098d9b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6098d9b6
Branch: refs/heads/two-dot-o-dev
Commit: 6098d9b60a2de306beb02d34f1bf34a42c67285f
Parents: 472ccaf
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Apr 23 16:20:12 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Apr 23 22:49:05 2015 -0600
----------------------------------------------------------------------
stack/core/pom.xml | 12 --
.../corepersistence/CpEntityManager.java | 29 +--
.../corepersistence/CpEntityManagerFactory.java | 21 +-
.../corepersistence/CpRelationManager.java | 71 +++++--
.../usergrid/corepersistence/CpWalker.java | 4 +-
.../corepersistence/command/CommandBuilder.java | 111 +++++++++++
.../command/cursor/CursorSerializer.java | 86 +++++++++
.../command/cursor/RequestCursor.java | 45 +++++
.../command/cursor/ResponseCursor.java | 56 ++++++
.../command/read/AbstractCommand.java | 111 +++++++++++
.../command/read/CollectCommand.java | 36 ++++
.../corepersistence/command/read/Command.java | 56 ++++++
.../command/read/TraverseCommand.java | 30 +++
.../command/read/entity/EntityLoadCommand.java | 191 +++++++++++++++++++
.../read/graph/AbstractReadGraphCommand.java | 110 +++++++++++
.../read/graph/ReadGraphCollectionCommand.java | 49 +++++
.../read/graph/ReadGraphConnectionCommand.java | 49 +++++
.../results/AbstractGraphQueryExecutor.java | 127 ++++++++++++
.../results/CollectionGraphQueryExecutor.java | 60 ++++++
.../CollectionResultsLoaderFactoryImpl.java | 1 -
.../results/ConnectionGraphQueryExecutor.java | 58 ++++++
.../rx/ApplicationObservable.java | 42 ++--
.../rx/EdgesFromSourceObservable.java | 4 +-
.../rx/EdgesToTargetObservable.java | 4 +-
.../usergrid/persistence/EntityManager.java | 7 +-
.../cassandra/EntityManagerImpl.java | 45 +++--
.../org/apache/usergrid/CoreApplication.java | 3 +
.../persistence/collection/EntitySet.java | 14 +-
.../serialization/impl/EntitySetImpl.java | 10 +-
.../persistence/core/rx/OrderedMerge.java | 28 +--
.../apache/usergrid/persistence/graph/Edge.java | 4 +
.../persistence/graph/SearchByEdgeType.java | 2 +-
.../graph/impl/SimpleSearchByEdgeType.java | 26 ++-
.../impl/stage/NodeDeleteListenerImpl.java | 5 +-
.../impl/shard/DirectedEdgeMeta.java | 7 +-
.../persistence/graph/GraphManagerLoadTest.java | 6 +-
.../graph/GraphManagerShardConsistencyIT.java | 3 +-
.../graph/GraphManagerStressTest.java | 8 +-
.../usergrid/persistence/model/entity/Id.java | 3 +
stack/corepersistence/pom.xml | 4 +-
.../usergrid/persistence/index/query/Query.java | 8 +
.../scenarios/ConnectionScenarios.scala | 2 +-
stack/pom.xml | 1 -
43 files changed, 1419 insertions(+), 130 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index d91208d..f12c0b8 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -473,18 +473,6 @@
</dependency>
<dependency>
- <groupId>com.netflix.rxjava</groupId>
- <artifactId>rxjava-core</artifactId>
- <version>${rx.version}</version>
- </dependency>
-
- <dependency>
- <groupId>com.netflix.rxjava</groupId>
- <artifactId>rxjava-math</artifactId>
- <version>${rx.version}</version>
- </dependency>
-
- <dependency>
<groupId>com.clearspring.analytics</groupId>
<artifactId>stream</artifactId>
<version>2.7.0</version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 789e640..038be44 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -34,6 +34,8 @@ import java.util.TreeSet;
import java.util.UUID;
import com.codahale.metrics.Meter;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.slf4j.Logger;
@@ -86,6 +88,7 @@ import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsE
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexScope;
@@ -178,8 +181,6 @@ public class CpEntityManager implements EntityManager {
private UUID applicationId;
private Application application;
- private CpEntityManagerFactory emf;
-
private ManagerCache managerCache;
private ApplicationScope applicationScope;
@@ -188,6 +189,10 @@ public class CpEntityManager implements EntityManager {
private CounterUtils counterUtils;
+
+ private EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private GraphManagerFactory graphManagerFactory;
+
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
private Timer aggCounterTimer;
@@ -220,22 +225,24 @@ public class CpEntityManager implements EntityManager {
}
@Override
- public void init( EntityManagerFactory emf, UUID applicationId ) {
+ public void init( final CassandraService cassandraService, final CounterUtils counterUtils, final MetricsFactory metricsFactory, final GraphManagerFactory graphManagerFactory, final EntityCollectionManagerFactory entityCollectionManagerFactory, final ManagerCache managerCache, UUID applicationId ) {
- Preconditions.checkNotNull( emf, "emf must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
- this.emf = ( CpEntityManagerFactory ) emf;
- this.managerCache = this.emf.getManagerCache();
+ this.managerCache = managerCache;
this.applicationId = applicationId;
applicationScope = CpNamingUtils.getApplicationScope( applicationId );
- this.cass = this.emf.getCassandraService();
- this.counterUtils = this.emf.getCounterUtils();
+ this.cass = cassandraService;
+ this.counterUtils = counterUtils;
//Timer Setup
- this.metricsFactory = this.emf.getMetricsFactory();
+ this.metricsFactory = metricsFactory;
+
+ this.graphManagerFactory = graphManagerFactory;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+
this.aggCounterTimer =this.metricsFactory.getTimer( CpEntityManager.class,
"cp.entity.get.aggregate.counters.timer" );
this.entCreateTimer =this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.create.timer" );
@@ -766,7 +773,7 @@ public class CpEntityManager implements EntityManager {
public RelationManager getRelationManager( EntityRef entityRef ) {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager rmi = new CpRelationManager();
- rmi.init( this, emf, applicationId, entityRef, null, metricsFactory );
+ rmi.init( this, applicationId, entityRef, null, metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory );
return rmi;
}
@@ -2885,7 +2892,7 @@ public class CpEntityManager implements EntityManager {
public void refreshIndex() {
// refresh factory indexes
- emf.refreshIndex();
+
// refresh this Entity Manager's application's index
EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index fe4d828..5b8e715 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -44,6 +44,7 @@ import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
@@ -57,6 +58,7 @@ import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.OrganizationAlreadyExistsException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
@@ -99,12 +101,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final OrgApplicationCache orgApplicationCache;
- private ManagerCache managerCache;
- private DataMigrationManager dataMigrationManager;
+ private final ManagerCache managerCache;
+ private final DataMigrationManager dataMigrationManager;
+ private final GraphManagerFactory graphManagerFactory;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private CassandraService cassandraService;
- private CounterUtils counterUtils;
- private Injector injector;
+ private final CassandraService cassandraService;
+ private final CounterUtils counterUtils;
+ private final Injector injector;
private final MetricsFactory metricsFactory;
public CpEntityManagerFactory(
@@ -115,8 +119,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.injector = injector;
this.managerCache = injector.getInstance( ManagerCache.class );
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
+ this.graphManagerFactory = injector.getInstance( GraphManagerFactory.class );
+ this.entityCollectionManagerFactory = injector.getInstance( EntityCollectionManagerFactory.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
+
this.orgApplicationCache = new OrgApplicationCacheImpl( this );
}
@@ -172,7 +179,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
EntityManager em = new CpEntityManager();
- em.init( this, applicationId );
+ em.init( cassandraService, counterUtils, metricsFactory, graphManagerFactory, entityCollectionManagerFactory, managerCache, applicationId );
return em;
}
@@ -427,7 +434,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
fromEntityId, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ));
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ));
Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
while ( iter.hasNext() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index da39ea9..edd352e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -33,6 +33,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.results.CollectionGraphQueryExecutor;
+import org.apache.usergrid.corepersistence.results.ConnectionGraphQueryExecutor;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
import org.apache.usergrid.corepersistence.results.QueryExecutor;
@@ -61,6 +63,8 @@ import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -71,6 +75,7 @@ import org.apache.usergrid.persistence.geo.EntityLocationRef;
import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
@@ -106,6 +111,7 @@ import org.apache.usergrid.utils.MapUtils;
import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import me.prettyprint.hector.api.Keyspace;
@@ -167,9 +173,9 @@ public class CpRelationManager implements RelationManager {
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
- private CpEntityManagerFactory emf;
-
private ManagerCache managerCache;
+ private EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private GraphManagerFactory graphManagerFactory;
private EntityManager em;
@@ -196,15 +202,16 @@ public class CpRelationManager implements RelationManager {
public CpRelationManager init(
- EntityManager em,
- CpEntityManagerFactory emf,
+ EntityManager em,
UUID applicationId,
EntityRef headEntity,
IndexBucketLocator indexBucketLocator,
- MetricsFactory metricsFactory) {
+ MetricsFactory metricsFactory,
+ ManagerCache managerCache,
+ EntityCollectionManagerFactory entityCollectionManagerFactory,
+ GraphManagerFactory graphManagerFactory ) {
Assert.notNull( em, "Entity manager cannot be null" );
- Assert.notNull( emf, "Entity manager factory cannot be null" );
Assert.notNull( applicationId, "Application Id cannot be null" );
Assert.notNull( headEntity, "Head entity cannot be null" );
Assert.notNull( headEntity.getUuid(), "Head entity uuid cannot be null" );
@@ -213,10 +220,9 @@ public class CpRelationManager implements RelationManager {
//Assert.notNull( indexBucketLocator, "indexBucketLocator cannot be null" );
this.em = em;
- this.emf = emf;
this.applicationId = applicationId;
this.headEntity = headEntity;
- this.managerCache = emf.getManagerCache();
+ this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence
@@ -232,6 +238,8 @@ public class CpRelationManager implements RelationManager {
// load the Core Persistence version of the head entity as well
this.headEntityScope = getCollectionScopeNameFromEntityType(
applicationScope.getApplication(), headEntity.getType());
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.graphManagerFactory = graphManagerFactory;
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading head entity {}:{} from scope\n app {}\n owner {}\n name {}",
@@ -331,7 +339,7 @@ public class CpRelationManager implements RelationManager {
public Observable<Edge> call( final String edgeType ) {
return gm.loadEdgesToTarget(
new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
} );
@@ -400,7 +408,7 @@ public class CpRelationManager implements RelationManager {
public Observable<Edge> call( final String etype ) {
return gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
cpHeadEntity.getId(), etype, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
} )
@@ -516,7 +524,7 @@ public class CpRelationManager implements RelationManager {
CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
System.currentTimeMillis(),
SearchByEdgeType.Order.DESCENDING,
- null ) ); // last
+ Optional.<Edge>absent() ) ); // last
Iterator<Edge> iterator = edgesToTarget.toBlockingObservable().getIterator();
int count = 0;
@@ -541,7 +549,7 @@ public class CpRelationManager implements RelationManager {
CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
System.currentTimeMillis(),
SearchByEdgeType.Order.DESCENDING,
- null ) ); // last
+ Optional.<Edge>absent() ) ); // last
int count = edgesFromSource.take( 2 ).count().toBlocking().last();
@@ -927,6 +935,21 @@ public class CpRelationManager implements RelationManager {
+ "' of " + headEntity.getType() + ":" + headEntity .getUuid() );
}
+ query.setEntityType( collection.getType() );
+ query = adjustQuery( query );
+
+ /**
+ * It's a graph query, execute the graph query executor.
+ *
+ * TODO refactor all of this away into commands
+ */
+ if(query.isGraphQuery()){
+ final QueryExecutor executor = new CollectionGraphQueryExecutor( this.entityCollectionManagerFactory,graphManagerFactory, applicationScope, headEntity, query.getCursor(), collName, query.getLimit() );
+ return executor.next();
+ }
+
+
+
final IndexScope indexScope = new IndexScopeImpl(
cpHeadEntity.getId(),
CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
@@ -939,9 +962,6 @@ public class CpRelationManager implements RelationManager {
indexScope.getOwner().toString(), indexScope.getName() );
- query.setEntityType( collection.getType() );
- query = adjustQuery( query );
-
final CollectionResultsLoaderFactoryImpl resultsLoaderFactory = new CollectionResultsLoaderFactoryImpl( managerCache );
@@ -1362,6 +1382,21 @@ public class CpRelationManager implements RelationManager {
headEntity = em.validate( headEntity );
+
+ query = adjustQuery( query );
+
+
+ /**
+ * It's a graph query, execute the graph query executor.
+ *
+ * TODO refactor all of this away into commands
+ */
+ if(query.isGraphQuery()){
+ final QueryExecutor executor = new ConnectionGraphQueryExecutor( this.entityCollectionManagerFactory,graphManagerFactory, applicationScope, headEntity, query.getCursor(), connection, query.getLimit() );
+ return executor.next();
+ }
+
+
final IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(),
CpNamingUtils.getConnectionScopeName( connection ) );
@@ -1373,7 +1408,6 @@ public class CpRelationManager implements RelationManager {
indexScope.getOwner().toString(), indexScope.getName(), searchTypes
} );
- query = adjustQuery( query );
final ConnectionResultsLoaderFactoryImpl resultsLoaderFactory = new ConnectionResultsLoaderFactoryImpl( managerCache,
headEntity, connection );
@@ -1381,9 +1415,6 @@ public class CpRelationManager implements RelationManager {
final QueryExecutor executor = new ElasticSearchQueryExecutor(resultsLoaderFactory, ei, applicationScope, indexScope, searchTypes, query);
return executor.next();
-// CandidateResults crs = ei.search( indexScope, searchTypes, query );
-
-// return buildConnectionResults( indexScope, query, crs, connection );
}
@@ -1455,7 +1486,7 @@ public class CpRelationManager implements RelationManager {
private CpRelationManager getRelationManager( EntityRef headEntity ) {
CpRelationManager rmi = new CpRelationManager();
- rmi.init( em, emf, applicationId, headEntity, null, metricsFactory);
+ rmi.init( em, applicationId, headEntity, null, metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory);
return rmi;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index 4b902d8..b2354a6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@ -31,6 +31,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import com.google.common.base.Optional;
+
import rx.Observable;
import rx.functions.Action1;
import rx.functions.Func1;
@@ -111,7 +113,7 @@ public class CpWalker {
logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
return gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- applicationId, edgeType, Long.MAX_VALUE, order , null ) );
+ applicationId, edgeType, Long.MAX_VALUE, order , Optional.<Edge>absent() ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
new file mode 100644
index 0000000..de76f50
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command;
+
+
+import org.apache.usergrid.corepersistence.command.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.command.cursor.ResponseCursor;
+import org.apache.usergrid.corepersistence.command.read.CollectCommand;
+import org.apache.usergrid.corepersistence.command.read.Command;
+import org.apache.usergrid.corepersistence.command.read.TraverseCommand;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * A builder that will allow us to build a traversal command for execution
+ */
+public class CommandBuilder {
+
+
+ private final ApplicationScope applicationScope;
+ private final RequestCursor requestCursor;
+ private final ResponseCursor responseCursor;
+ private final int limit;
+
+ private int count = 0;
+
+ private Observable<Id> currentObservable;
+
+
+ /**
+ * Our first pass, where we implement our start point as an Id until we can use this to perform our entire
+ * traversal. Eventually as we untangle the existing Query service nightmare, the sourceId will be remove and should
+ * only be traversed from the root application
+ */
+ public CommandBuilder( final ApplicationScope applicationScope, final Id sourceId,
+ final Optional<String> requestCursor, final int limit ) {
+
+ this.applicationScope = applicationScope;
+ this.limit = limit;
+
+ //set the request cursor
+ this.requestCursor = new RequestCursor( requestCursor );
+
+ //set the response cursor
+ this.responseCursor = new ResponseCursor();
+
+
+ this.currentObservable = Observable.just( sourceId );
+ }
+
+
+ /**
+ * Add a read command that will read Ids and produce Ids. This is an intermediate traversal operations
+ */
+ public CommandBuilder withTraverseCommand( final TraverseCommand traverseCommand ) {
+
+ setState( traverseCommand );
+
+ this.currentObservable = currentObservable.compose( traverseCommand );
+
+ return this;
+ }
+
+
+ /**
+ * Build the final collection step, and
+ */
+ public <T> Observable<T> build( final CollectCommand<T> collectCommand ) {
+ setState( collectCommand );
+
+ collectCommand.setLimit( limit );
+
+ return currentObservable.compose( collectCommand );
+ }
+
+
+ /**
+ * Set the id of the state
+ * @param command
+ */
+ private void setState( Command<?> command ) {
+ command.setId( count );
+ //done for clarity
+ count++;
+
+ command.setCursorCaches( requestCursor, responseCursor );
+ command.setApplicationScope( applicationScope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java
new file mode 100644
index 0000000..b45e7da
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.cursor;
+
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.core.Base64Variant;
+import com.fasterxml.jackson.core.Base64Variants;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+
+
+/**
+ * A utility to serialize objects to/from cursors
+ */
+public class CursorSerializer {
+
+
+ private static final SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+
+ private static final Base64Variant VARIANT = Base64Variants.MODIFIED_FOR_URL;
+
+
+ /**
+ * Serialize the serializable object as a cursor
+ */
+ public static String asCursor( final Serializable cursor ) {
+
+ try {
+ return MAPPER.writer( VARIANT ).writeValueAsString( cursor );
+ }
+ catch ( JsonProcessingException e ) {
+ throw new CursorParseException( "Unable to serialize cursor", e );
+ }
+ }
+
+
+ /**
+ * Deserialize from the cursor
+ * @param cursor
+ * @return
+ * @throws CursorParseException
+ */
+ public <T extends Serializable> T fromCursor( final String cursor, final Class<T> cursorClass ) throws CursorParseException {
+ try {
+
+ final JsonParser parser = MAPPER.getFactory().createParser( cursor );
+ return MAPPER.reader( VARIANT ).readValue( parser, cursorClass);
+ }
+ catch ( Exception e ) {
+ throw new CursorParseException( "Unable to serialize cursor", e );
+ }
+ }
+
+
+ /**
+ * Thrown when we can't parse a cursor
+ */
+ public static class CursorParseException extends RuntimeException {
+ public CursorParseException( final String message, final Throwable cause ) {
+ super( message, cause );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java
new file mode 100644
index 0000000..60d54ff
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.cursor;
+
+
+import java.io.Serializable;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A cursor that has been passed in with our request. Adds utils for parsing values
+ */
+public class RequestCursor {
+
+ public RequestCursor(final Optional<String> cursor){
+
+ }
+
+
+ /**
+ * Get the cursor with the specified id
+ */
+ public <T extends Serializable> T getCursor( final int id, final Class<T> cursorType ) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java
new file mode 100644
index 0000000..02aae34
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.cursor;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * A cursor used in rendering a response
+ */
+public class ResponseCursor {
+
+
+
+ /**
+ * We use a map b/c some indexes might be skipped
+ */
+ private Map<Integer, ? super Serializable> cursors = new HashMap<>();
+
+ /**
+ * Set the possible cursor value into the index. DOES NOT parse the cursor. This is intentional for performance
+ */
+ public <T extends Serializable> void setCursor( final int id, final T cursor ) {
+ cursors.put( id, cursor );
+ }
+
+
+ private void ensureCapacity() {
+
+ }
+
+
+ public String encodeAsString() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java
new file mode 100644
index 0000000..59e1848
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read;
+
+
+import java.io.Serializable;
+
+import javax.xml.ws.Response;
+
+import org.apache.usergrid.corepersistence.command.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.command.cursor.ResponseCursor;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * Basic functionality for our commands to handle cursor IO
+ */
+public abstract class AbstractCommand<T, C extends Serializable> implements Command<T> {
+
+ private int id;
+ /**
+ * The cache of the cursor that was set when the read was started
+ */
+ private RequestCursor readCache;
+
+ /**
+ * The current state of the write cache. Gets updated as we traverse the observables
+ */
+ private ResponseCursor writeCache;
+
+
+ /**
+ * The applicationScope
+ */
+ protected ApplicationScope applicationScope;
+
+
+ @Override
+ public void setId( final int id ) {
+ this.id = id;
+ }
+
+
+ @Override
+ public void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache ) {
+ this.readCache = readCache;
+ this.writeCache = writeCache;
+ }
+
+
+ @Override
+ public void setApplicationScope( final ApplicationScope applicationScope ) {
+ this.applicationScope = applicationScope;
+ }
+
+
+ /**
+ * Return the parsed value of the cursor from the last request, if it exists
+ */
+ protected Optional<C> getCursor() {
+ final C cursor = readCache.getCursor( id, getCursorClass() );
+
+ return Optional.fromNullable( cursor );
+ }
+
+
+
+
+
+ /**
+ * Set the cursor value into the new cursor write cache
+ * @param newValue
+ */
+ protected void setCursor(final C newValue){
+ writeCache.setCursor( id, newValue );
+ }
+
+
+ /**
+ * Generate our state as a cursor
+ * @return
+ */
+ protected String generateCursor(){
+ return writeCache.encodeAsString();
+ }
+
+ /**
+ * Return the class to be used when parsing the cursor
+ */
+ protected abstract Class<C> getCursorClass();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java
new file mode 100644
index 0000000..c2ad931
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read;
+
+
+/**
+ * A command that is used to reduce our stream of results into a final output
+ * @param <T>
+ */
+public interface CollectCommand<T> extends Command<T>{
+
+ /**
+ * Set the prefered result size for the command
+ * @param resultSize
+ */
+ void setLimit( final int resultSize );
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java
new file mode 100644
index 0000000..ada47d2
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read;
+
+
+import org.apache.usergrid.corepersistence.command.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.command.cursor.ResponseCursor;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Interface for a read command. This takes an input of Id, performs some operation, and emits Id for further
+ * processing
+ */
+public interface Command<T> extends Observable.Transformer<Id, T> {
+
+
+ /**
+ * Set the id of this command in it's execution environment
+ */
+ void setId( final int id );
+
+ /**
+ * Set the cursor cache into the command
+ *
+ * @param readCache Set the cache that was used in the request
+ * @param writeCache Set the cache to be used when writing the results
+ */
+ void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache );
+
+ /**
+ * Set the application scope of the command
+ * @param applicationScope
+ */
+ void setApplicationScope(final ApplicationScope applicationScope);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java
new file mode 100644
index 0000000..c8fe2f8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read;
+
+
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Traverses edges in the graph. Either by query or graph traversal. Take an observable of ids, and emits
+ * an observable of ids
+ */
+public interface TraverseCommand extends Command<Id> {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
new file mode 100644
index 0000000..f4de9e0
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.entity;
+
+
+import java.io.Serializable;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
+import org.apache.usergrid.corepersistence.command.read.CollectCommand;
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+import rx.functions.Func1;
+import rx.observables.GroupedObservable;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
+
+
+/**
+ * Loads entities from a set of Ids.
+ *
+ * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
+ */
+public class EntityLoadCommand extends AbstractCommand<Results, Serializable> implements CollectCommand<Results> {
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+ //TODO get rid of this when merged into 2.0 dev
+ private final ApplicationScope applicationScope;
+ private int resultSize;
+
+
+ public EntityLoadCommand( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final ApplicationScope applicationScope ) {
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.applicationScope = applicationScope;
+ }
+
+
+ @Override
+ protected Class<Serializable> getCursorClass() {
+ return null;
+ }
+
+
+ @Override
+ public Observable<Results> call( final Observable<? extends Id> observable ) {
+
+
+ /**
+ * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
+ * objects
+ */
+
+
+ return observable.buffer( resultSize ).flatMap( new Func1<List<? extends Id>, Observable<Results>>() {
+ @Override
+ public Observable<Results> call( final List<? extends Id> ids ) {
+
+ return Observable.from( ids )
+ //group them by type so we can load them, in 2.0 dev this step will be removed
+ .groupBy( new Func1<Id, String>() {
+ @Override
+ public String call( final Id id ) {
+ return id.getType();
+ }
+ } )
+
+ //take all our groups and load them as id sets
+
+ .flatMap( new Func1<GroupedObservable<String, Id>, Observable<EntitySet>>() {
+ @Override
+ public Observable<EntitySet> call(
+ final GroupedObservable<String, Id> stringIdGroupedObservable ) {
+
+
+ final String entityType = stringIdGroupedObservable.getKey();
+
+ final CollectionScope collectionScope =
+ getCollectionScopeNameFromEntityType( applicationScope.getApplication(), entityType );
+
+
+ return stringIdGroupedObservable.toList()
+ .flatMap( new Func1<List<Id>, Observable<EntitySet>>() {
+ @Override
+ public Observable<EntitySet> call(
+ final List<Id> ids ) {
+
+ final EntityCollectionManager ecm =
+ entityCollectionManagerFactory
+ .createCollectionManager( collectionScope );
+ return ecm.load( ids );
+ }
+ } );
+ }
+ } )
+ //emit our groups of entities as a stream of entities
+ .flatMap( new Func1<EntitySet, Observable<org.apache.usergrid.persistence.Entity>>() {
+ @Override
+ public Observable<org.apache.usergrid.persistence.Entity> call( final EntitySet entitySet ) {
+ //emit our entities, and filter out deleted entites
+ return Observable.from( entitySet.getEntities() ).map(
+ new Func1<MvccEntity, org.apache.usergrid.persistence.Entity>() {
+
+ @Override
+ public org.apache.usergrid.persistence.Entity call( final MvccEntity mvccEntity ) {
+ return mapEntity( mvccEntity );
+ }
+ } )
+ //filter null entities
+ .filter( new Func1<org.apache.usergrid.persistence.Entity, Boolean>() {
+ @Override
+ public Boolean call( final org.apache.usergrid.persistence.Entity entity ) {
+ return entity == null;
+ }
+ } );
+ }
+ } )
+
+ //convert them to a list, then map them into results
+ .toList().map( new Func1<List<org.apache.usergrid.persistence.Entity>, Results>() {
+ @Override
+ public Results call( final List<org.apache.usergrid.persistence.Entity> entities ) {
+ final Results results = Results.fromEntities( entities );
+ results.setCursor( generateCursor() );
+
+ return results;
+ }
+ } );
+ }
+ } );
+ }
+
+
+
+ /**
+ * Map a new cp entity to an old entity. May be null if not present
+ */
+ private org.apache.usergrid.persistence.Entity mapEntity( final MvccEntity mvccEntity ) {
+ if ( !mvccEntity.getEntity().isPresent() ) {
+ return null;
+ }
+
+
+ final Entity cpEntity = mvccEntity.getEntity().get();
+ final Id entityId = cpEntity.getId();
+
+ org.apache.usergrid.persistence.Entity entity =
+ EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
+
+ Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
+ entity.addProperties( entityMap );
+
+ return entity;
+ }
+
+
+ @Override
+ public void setLimit( final int resultSize ) {
+ this.resultSize = resultSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
new file mode 100644
index 0000000..141fb06
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.graph;
+
+
+import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
+import org.apache.usergrid.corepersistence.command.read.TraverseCommand;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Func1;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
+
+
+/**
+ * Command for reading graph edges
+ */
+public abstract class AbstractReadGraphCommand extends AbstractCommand<Id, Edge> implements TraverseCommand {
+
+ private final GraphManagerFactory graphManagerFactory;
+
+
+ /**
+ * Create a new instance of our command
+ * @param graphManagerFactory
+ */
+ public AbstractReadGraphCommand( final GraphManagerFactory graphManagerFactory ) {
+ this.graphManagerFactory = graphManagerFactory;
+ }
+
+
+ @Override
+ public Observable<Id> call( final Observable<? extends Id> observable ) {
+
+ //get the graph manager
+ final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //set our our constant state
+ final Optional<Edge> startFromCursor = getCursor();
+
+ final String edgeName = getEdgeTypeName();
+
+
+ //return all ids that are emitted from this edge
+ return observable.flatMap( new Func1<Id, Observable<Id>>() {
+
+ @Override
+ public Observable<Id> call( final Id id ) {
+
+ final SimpleSearchByEdgeType search = new SimpleSearchByEdgeType(id,edgeName, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, startFromCursor );
+
+ /**
+ * TODO, pass a message with pointers to our cursor values to be generated later
+ */
+ return graphManager.loadEdgesFromSource( search ).doOnNext( new Action1<Edge>() {
+ @Override
+ public void call( final Edge edge ) {
+ setCursor( edge );
+ }
+ } ).map( new Func1<Edge, Id>() {
+ @Override
+ public Id call( final Edge edge ) {
+ return edge.getTargetNode();
+ }
+ } );
+ }
+ } );
+ }
+
+
+ @Override
+ protected Class<Edge> getCursorClass() {
+ return Edge.class;
+ }
+
+
+
+ /**
+ * Get the edge type name we should use when traversing
+ * @return
+ */
+ protected abstract String getEdgeTypeName();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
new file mode 100644
index 0000000..9ccb969
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.graph;
+
+
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
+
+
+/**
+ * Command for reading graph edges on a collection
+ */
+public class ReadGraphCollectionCommand extends AbstractReadGraphCommand {
+
+ private final String collectionName;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ public ReadGraphCollectionCommand( final GraphManagerFactory graphManagerFactory, final String collectionName ) {
+ super( graphManagerFactory );
+ this.collectionName = collectionName;
+ }
+
+
+ @Override
+ protected String getEdgeTypeName() {
+ return getCollectionScopeNameFromCollectionName( collectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
new file mode 100644
index 0000000..adebd45
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.graph;
+
+
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
+
+
+/**
+ * Command for reading graph edges on a connection
+ */
+public class ReadGraphConnectionCommand extends AbstractReadGraphCommand {
+
+ private final String connectionName;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ public ReadGraphConnectionCommand( final GraphManagerFactory graphManagerFactory, final String connectionName ) {
+ super( graphManagerFactory );
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected String getEdgeTypeName() {
+ return getConnectionScopeName( connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
new file mode 100644
index 0000000..0bea013
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.usergrid.corepersistence.command.CommandBuilder;
+import org.apache.usergrid.corepersistence.command.read.entity.EntityLoadCommand;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * This class is a nasty hack to bridge 2.0 observables into 1.0 iterators DO NOT use this as a model for moving
+ * forward, pandas will die.
+ */
+public abstract class AbstractGraphQueryExecutor implements QueryExecutor {
+
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final ApplicationScope applicationScope;
+ private final Id sourceId;
+ private final int limit;
+
+ private final Optional<String> requestCursor;
+
+ private Iterator<Results> observableIterator;
+
+
+ public AbstractGraphQueryExecutor( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final ApplicationScope applicationScope, final EntityRef source,
+ final String cursor, final int limit ) {
+
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.applicationScope = applicationScope;
+ this.limit = limit;
+ this.sourceId = new SimpleId( source.getUuid(), source.getType() );
+
+ this.requestCursor = Optional.fromNullable( cursor );
+ }
+
+
+ @Override
+ public Iterator<Results> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+
+ //hasn't been set up yet, run through our first setup
+ if ( observableIterator == null ) {
+ //assign them to an interator. this now uses an internal buffer with backpressure, so we won't load all
+ // results
+ //set up our command builder
+ final CommandBuilder commandBuilder = new CommandBuilder( applicationScope, sourceId, requestCursor, limit );
+
+
+ addTraverseCommand( commandBuilder );
+
+ //construct our results to be observed later. This is a cold observable
+ final Observable<Results> resultsObservable =
+ commandBuilder.build( new EntityLoadCommand( entityCollectionManagerFactory, applicationScope ) );
+
+ this.observableIterator = resultsObservable.toBlocking().getIterator();
+ }
+
+
+ //see if our current results are not null
+ return observableIterator.hasNext();
+ }
+
+
+ @Override
+ public Results next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more results present" );
+ }
+
+ final Results results = observableIterator.next();
+
+ //ugly and tight coupling, but we don't have a choice until we finish some refactoring
+ results.setQueryExecutor( this );
+
+ return results;
+ }
+
+
+ @Override
+ public void remove() {
+ throw new RuntimeException( "Remove not implemented!!" );
+ }
+
+
+ /**
+ * Add the traverse command to the graph
+ */
+ protected abstract void addTraverseCommand( final CommandBuilder commandBuilder );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
new file mode 100644
index 0000000..651c27e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import org.apache.usergrid.corepersistence.command.CommandBuilder;
+import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphCollectionCommand;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import com.google.common.base.Preconditions;
+
+
+/**
+ * Create The collection graph query executor
+ */
+public class CollectionGraphQueryExecutor extends AbstractGraphQueryExecutor {
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final String collectionName;
+
+
+ public CollectionGraphQueryExecutor( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final GraphManagerFactory graphManagerFactory,
+ final ApplicationScope applicationScope, final EntityRef source,
+ final String cursor, final String connectionName, final int limit ) {
+
+ super( entityCollectionManagerFactory, applicationScope, source, cursor, limit );
+ this.graphManagerFactory = graphManagerFactory;
+
+ Preconditions.checkNotNull( connectionName, "connectionName is required on the query" );
+ this.collectionName = connectionName;
+ }
+
+
+ @Override
+ protected void addTraverseCommand( final CommandBuilder commandBuilder ) {
+ //set the traverse command from the source Id to the connect name
+ commandBuilder.withTraverseCommand( new ReadGraphCollectionCommand( graphManagerFactory, collectionName ) );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
index b79700b..4b43142 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
@@ -48,7 +48,6 @@ public class CollectionResultsLoaderFactoryImpl implements ResultsLoaderFactory
verifier = new CollectionRefsVerifier();
}
else if ( resultsLevel == Query.Level.IDS ) {
-// verifier = new RefsVerifier();
verifier = new IdsVerifier();
}
else {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
new file mode 100644
index 0000000..61726b7
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import org.apache.usergrid.corepersistence.command.CommandBuilder;
+import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphConnectionCommand;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import com.google.common.base.Preconditions;
+
+
+public class ConnectionGraphQueryExecutor extends AbstractGraphQueryExecutor {
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final String connectionName;
+
+
+ public ConnectionGraphQueryExecutor( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final GraphManagerFactory graphManagerFactory,
+ final ApplicationScope applicationScope, final EntityRef source,
+ final String cursor, final String connectionType, final int limit) {
+
+ super( entityCollectionManagerFactory, applicationScope, source, cursor, limit );
+ this.graphManagerFactory = graphManagerFactory;
+
+ Preconditions.checkNotNull(connectionType, "connectionType is required on the query" );
+ this.connectionName = connectionType;
+ }
+
+
+
+ @Override
+ protected void addTraverseCommand( final CommandBuilder commandBuilder ) {
+ //set the traverse command from the source Id to the connect name
+ commandBuilder.withTraverseCommand( new ReadGraphConnectionCommand( graphManagerFactory, connectionName ) );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
index 6019bca..988cd3b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
@@ -39,6 +39,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
import rx.Observable;
import rx.functions.Func1;
@@ -89,37 +91,37 @@ public class ApplicationObservable {
//we have app infos. For each of these app infos, we have to load the application itself
Observable<Id> appIds = gm.loadEdgesFromSource(
new SimpleSearchByEdgeType( rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- null ) ).flatMap( new Func1<Edge, Observable<Id>>() {
+ Optional.<Edge>absent() ) ).flatMap( new Func1<Edge, Observable<Id>>() {
@Override
public Observable<Id> call( final Edge edge ) {
//get the app info and load it
final Id appInfo = edge.getTargetNode();
return collectionManager.load( appInfo )
- //filter out null entities
- .filter( new Func1<Entity, Boolean>() {
- @Override
- public Boolean call( final Entity entity ) {
- if ( entity == null ) {
- logger.warn( "Encountered a null application info for id {}", appInfo );
- return false;
- }
-
- return true;
+ //filter out null entities
+ .filter( new Func1<Entity, Boolean>() {
+ @Override
+ public Boolean call( final Entity entity ) {
+ if ( entity == null ) {
+ logger.warn( "Encountered a null application info for id {}", appInfo );
+ return false;
}
- } )
- //get the id from the entity
- .map( new Func1<org.apache.usergrid.persistence.model.entity.Entity, Id>() {
+ return true;
+ }
+ } )
+ //get the id from the entity
+ .map( new Func1<org.apache.usergrid.persistence.model.entity.Entity, Id>() {
- @Override
- public Id call( final org.apache.usergrid.persistence.model.entity.Entity entity ) {
- final UUID uuid = ( UUID ) entity.getField( "applicationUuid" ).getValue();
+ @Override
+ public Id call( final org.apache.usergrid.persistence.model.entity.Entity entity ) {
- return CpNamingUtils.generateApplicationId( uuid );
- }
- } );
+ final UUID uuid = ( UUID ) entity.getField( "applicationUuid" ).getValue();
+
+ return CpNamingUtils.generateApplicationId( uuid );
+ }
+ } );
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java
index d3e2ee5..b23886b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java
@@ -31,6 +31,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
import rx.Observable;
import rx.functions.Func1;
@@ -56,7 +58,7 @@ public class EdgesFromSourceObservable {
logger.debug( "Loading edges of edgeType {} from {}", edgeType, sourceNode );
return gm.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java
index c5dc54d..3130a72 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java
@@ -31,6 +31,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
import rx.Observable;
import rx.functions.Func1;
@@ -56,7 +58,7 @@ public class EdgesToTargetObservable {
logger.debug( "Loading edges of edgeType {} to {}", edgeType, targetNode);
return gm.loadEdgesToTarget( new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index 65fac8d..645618c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -17,6 +17,11 @@
package org.apache.usergrid.persistence;
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.persistence.cassandra.CounterUtils;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.index.query.Query;
import java.nio.ByteBuffer;
import java.util.Collection;
@@ -698,7 +703,7 @@ public interface EntityManager {
*/
void deleteIndex();
- public void init( EntityManagerFactory emf, UUID applicationId);
+ public void init( final CassandraService cassandraService, final CounterUtils counterUtils, final MetricsFactory metricsFactory, final GraphManagerFactory graphManagerFactory, final EntityCollectionManagerFactory entityCollectionManagerFactory, final ManagerCache managerCache, UUID applicationId);
/** For testing purposes */
public void flushManagerCaches();
[10/20] incubator-usergrid git commit: Fixes wiring and updates code
to be compatible with 2.0-dev
Posted by to...@apache.org.
Fixes wiring and updates code to be compatible with 2.0-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/83174e8b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/83174e8b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/83174e8b
Branch: refs/heads/two-dot-o-dev
Commit: 83174e8bf6675476e21a8d9e47ae0eadff6cb7a3
Parents: ec3a710
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 10:28:45 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 10:28:45 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 13 ++-
.../corepersistence/CpRelationManager.java | 34 ++++--
.../command/read/entity/EntityLoadCommand.java | 113 +++++--------------
.../read/graph/AbstractReadGraphCommand.java | 2 +-
.../migration/AppInfoMigrationPlugin.java | 6 +-
.../results/CollectionGraphQueryExecutor.java | 6 +-
.../rx/impl/AllApplicationsObservableImpl.java | 7 +-
.../corepersistence/util/CpNamingUtils.java | 32 +++---
.../org/apache/usergrid/persistence/Query.java | 5 +-
.../serialization/impl/EdgesObservableImpl.java | 6 +-
.../usergrid/persistence/index/IndexFig.java | 2 -
11 files changed, 99 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 6ba71e2..f06cf40 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -50,6 +50,7 @@ import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
@@ -61,6 +62,7 @@ import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsExcept
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
@@ -116,6 +118,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final ApplicationIdCache applicationIdCache;
private ManagerCache managerCache;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final GraphManagerFactory graphManagerFactory;
private CassandraService cassandraService;
private CounterUtils counterUtils;
@@ -125,7 +129,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final AsyncIndexService indexService;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
- final Injector injector) {
+ final Injector injector ) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
@@ -137,6 +141,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.indexService = injector.getInstance( AsyncIndexService.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
+ this.entityCollectionManagerFactory = injector.getInstance( EntityCollectionManagerFactory.class );
+ this.graphManagerFactory = injector.getInstance( GraphManagerFactory.class );
+
}
@@ -192,8 +199,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId,
- entityCollectionManagerFactory, graphManagerFactory );
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory,
+ entityCollectionManagerFactory, graphManagerFactory, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 8e2fdd7..86ba300 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -31,7 +31,9 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.index.AsyncIndexService;
+import org.apache.usergrid.corepersistence.results.CollectionGraphQueryExecutor;
import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.results.ConnectionGraphQueryExecutor;
import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
import org.apache.usergrid.corepersistence.results.QueryExecutor;
@@ -51,7 +53,6 @@ import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -635,19 +636,28 @@ public class CpRelationManager implements RelationManager {
}
- final SearchEdge searchEdge = createCollectionSearchEdge( cpHeadEntity.getId(), collName );
- final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ query.setEntityType( collection.getType() );
+ query = adjustQuery( query );
- final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
- logger.debug( "Searching scope {}", searchEdge );
+ if ( query.isGraphSearch() ) {
+ QueryExecutor executor =
+ new CollectionGraphQueryExecutor( entityCollectionManagerFactory, graphManagerFactory, applicationScope,
+ headEntity, query.getOffsetCursor(), collName, query.getLimit() );
- query.setEntityType( collection.getType() );
- query = adjustQuery( query );
+ return executor.next();
+ }
+ final SearchEdge searchEdge = createCollectionSearchEdge( cpHeadEntity.getId(), collName );
+
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
+
+ final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
+
+ logger.debug( "Searching scope {}", searchEdge );
final CollectionResultsLoaderFactoryImpl resultsLoaderFactory =
new CollectionResultsLoaderFactoryImpl( managerCache );
@@ -909,6 +919,16 @@ public class CpRelationManager implements RelationManager {
query = adjustQuery( query );
+
+
+ if ( query.isGraphSearch() ) {
+ QueryExecutor executor =
+ new ConnectionGraphQueryExecutor( entityCollectionManagerFactory, graphManagerFactory, applicationScope,
+ headEntity, query.getOffsetCursor(), connection, query.getLimit() );
+
+ return executor.next();
+ }
+
final ConnectionResultsLoaderFactoryImpl resultsLoaderFactory =
new ConnectionResultsLoaderFactoryImpl( managerCache, headEntity, connection );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
index f4de9e0..08193ef 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.corepersistence.command.read.entity;
import java.io.Serializable;
-import java.util.List;
import java.util.Map;
import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
@@ -29,7 +28,6 @@ import org.apache.usergrid.corepersistence.command.read.CollectCommand;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.persistence.EntityFactory;
import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.collection.EntitySet;
@@ -39,10 +37,6 @@ import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
-import rx.functions.Func1;
-import rx.observables.GroupedObservable;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
/**
@@ -73,7 +67,7 @@ public class EntityLoadCommand extends AbstractCommand<Results, Serializable> im
@Override
- public Observable<Results> call( final Observable<? extends Id> observable ) {
+ public Observable<Results> call( final Observable<Id> observable ) {
/**
@@ -81,90 +75,39 @@ public class EntityLoadCommand extends AbstractCommand<Results, Serializable> im
* objects
*/
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ final Observable<EntitySet> entitySetObservable = observable.buffer( resultSize ).flatMap(
+ bufferedIds -> Observable.just( bufferedIds ).flatMap( ids -> entityCollectionManager.load( ids ) ) );
+
+
+ return entitySetObservable
+
+ .flatMap( entitySet -> {
+
+ //get our entites and filter missing ones, then collect them into a results object
+ final Observable<MvccEntity> mvccEntityObservable = Observable.from( entitySet.getEntities() );
+
+ //convert them to our old entity model, then filter nulls, meaning they weren't found
+ return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter(
+ entity -> entity == null )
+
+ //convert them to a list, then map them into results
+ .toList().map( entities -> {
+ final Results results = Results.fromEntities( entities );
+ results.setCursor( generateCursor() );
- return observable.buffer( resultSize ).flatMap( new Func1<List<? extends Id>, Observable<Results>>() {
- @Override
- public Observable<Results> call( final List<? extends Id> ids ) {
-
- return Observable.from( ids )
- //group them by type so we can load them, in 2.0 dev this step will be removed
- .groupBy( new Func1<Id, String>() {
- @Override
- public String call( final Id id ) {
- return id.getType();
- }
- } )
-
- //take all our groups and load them as id sets
-
- .flatMap( new Func1<GroupedObservable<String, Id>, Observable<EntitySet>>() {
- @Override
- public Observable<EntitySet> call(
- final GroupedObservable<String, Id> stringIdGroupedObservable ) {
-
-
- final String entityType = stringIdGroupedObservable.getKey();
-
- final CollectionScope collectionScope =
- getCollectionScopeNameFromEntityType( applicationScope.getApplication(), entityType );
-
-
- return stringIdGroupedObservable.toList()
- .flatMap( new Func1<List<Id>, Observable<EntitySet>>() {
- @Override
- public Observable<EntitySet> call(
- final List<Id> ids ) {
-
- final EntityCollectionManager ecm =
- entityCollectionManagerFactory
- .createCollectionManager( collectionScope );
- return ecm.load( ids );
- }
- } );
- }
- } )
- //emit our groups of entities as a stream of entities
- .flatMap( new Func1<EntitySet, Observable<org.apache.usergrid.persistence.Entity>>() {
- @Override
- public Observable<org.apache.usergrid.persistence.Entity> call( final EntitySet entitySet ) {
- //emit our entities, and filter out deleted entites
- return Observable.from( entitySet.getEntities() ).map(
- new Func1<MvccEntity, org.apache.usergrid.persistence.Entity>() {
-
- @Override
- public org.apache.usergrid.persistence.Entity call( final MvccEntity mvccEntity ) {
- return mapEntity( mvccEntity );
- }
- } )
- //filter null entities
- .filter( new Func1<org.apache.usergrid.persistence.Entity, Boolean>() {
- @Override
- public Boolean call( final org.apache.usergrid.persistence.Entity entity ) {
- return entity == null;
- }
- } );
- }
- } )
-
- //convert them to a list, then map them into results
- .toList().map( new Func1<List<org.apache.usergrid.persistence.Entity>, Results>() {
- @Override
- public Results call( final List<org.apache.usergrid.persistence.Entity> entities ) {
- final Results results = Results.fromEntities( entities );
- results.setCursor( generateCursor() );
-
- return results;
- }
+ return results;
} );
- }
- } );
+ } );
}
+ /**
+ * Map a new cp entity to an old entity. May be null if not present
+ */
- /**
- * Map a new cp entity to an old entity. May be null if not present
- */
private org.apache.usergrid.persistence.Entity mapEntity( final MvccEntity mvccEntity ) {
if ( !mvccEntity.getEntity().isPresent() ) {
return null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
index 141fb06..95b6c6b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
@@ -56,7 +56,7 @@ public abstract class AbstractReadGraphCommand extends AbstractCommand<Id, Edge>
@Override
- public Observable<Id> call( final Observable<? extends Id> observable ) {
+ public Observable<Id> call( final Observable<Id> observable ) {
//get the graph manager
final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
index 129f2f9..30955af 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/migration/AppInfoMigrationPlugin.java
@@ -18,6 +18,7 @@
*/
package org.apache.usergrid.corepersistence.migration;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
@@ -261,7 +262,7 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
Id rootAppId = systemAppScope.getApplication();
final SimpleSearchByEdgeType simpleSearchByEdgeType = new SimpleSearchByEdgeType(
- rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null);
+ rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent());
Observable<org.apache.usergrid.persistence.model.entity.Entity> entityObs =
gm.loadEdgesFromSource( simpleSearchByEdgeType )
@@ -284,7 +285,8 @@ public class AppInfoMigrationPlugin implements MigrationPlugin {
final SimpleSearchByEdgeType simpleSearchByEdgeType = new SimpleSearchByEdgeType(
- CpNamingUtils.generateApplicationId(CpNamingUtils.MANAGEMENT_APPLICATION_ID), edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null);
+ CpNamingUtils.generateApplicationId(CpNamingUtils.MANAGEMENT_APPLICATION_ID), edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ Optional.absent());
edgesObservable = edgesObservable !=null ? edgesObservable : gm.loadEdgesFromSource( simpleSearchByEdgeType );
return edgesObservable;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
index 651c27e..515fc2b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
@@ -42,13 +42,13 @@ public class CollectionGraphQueryExecutor extends AbstractGraphQueryExecutor {
public CollectionGraphQueryExecutor( final EntityCollectionManagerFactory entityCollectionManagerFactory,
final GraphManagerFactory graphManagerFactory,
final ApplicationScope applicationScope, final EntityRef source,
- final String cursor, final String connectionName, final int limit ) {
+ final String cursor, final String collectionName, final int limit ) {
super( entityCollectionManagerFactory, applicationScope, source, cursor, limit );
this.graphManagerFactory = graphManagerFactory;
- Preconditions.checkNotNull( connectionName, "connectionName is required on the query" );
- this.collectionName = connectionName;
+ Preconditions.checkNotNull( collectionName, "collectionName is required on the query" );
+ this.collectionName = collectionName;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
index 5e2f6b6..fbabbee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/AllApplicationsObservableImpl.java
@@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.utils.UUIDUtils;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import rx.Observable;
@@ -91,8 +92,8 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
//we have app infos. For each of these app infos, we have to load the application itself
Observable<ApplicationScope> appIds = gm.loadEdgesFromSource(
- new SimpleSearchByEdgeType( rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- null ) ).flatMap( new Func1<Edge, Observable<ApplicationScope>>() {
+ new SimpleSearchByEdgeType( rootAppId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ Optional.absent() ) ).flatMap( new Func1<Edge, Observable<ApplicationScope>>() {
@Override
public Observable<ApplicationScope> call( final Edge edge ) {
@@ -118,7 +119,7 @@ public class AllApplicationsObservableImpl implements AllApplicationsObservable
@Override
public ApplicationScope call( final Entity entity ) {
final UUID uuid = UUIDUtils.tryExtractUUID(
- entity.getField( Schema.PROPERTY_APPLICATION_ID ).getValue().toString());
+ entity.getField( Schema.PROPERTY_APPLICATION_ID ).getValue().toString() );
return getApplicationScope( uuid );
}
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 364b071..5472071 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.corepersistence.util;
import java.util.UUID;
-import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.entities.Application;
@@ -73,7 +72,6 @@ public class CpNamingUtils {
public static String TYPES_BY_UUID_MAP = "zzz_typesbyuuid_zzz";
-
/**
* Generate a standard edge name for our graph using the connection name. To be used only for searching. DO NOT use
* for creation. Use the createConnectionEdge instead.
@@ -99,7 +97,8 @@ public class CpNamingUtils {
* Get the index scope for the edge from the source
*/
public static IndexEdge generateScopeFromSource( final Edge edge ) {
- return new IndexEdgeImpl( edge.getSourceNode(), edge.getType(), SearchEdge.NodeType.SOURCE, edge.getTimestamp() );
+ return new IndexEdgeImpl( edge.getSourceNode(), edge.getType(), SearchEdge.NodeType.SOURCE,
+ edge.getTimestamp() );
}
@@ -107,7 +106,8 @@ public class CpNamingUtils {
* Get the index scope for the edge from the source
*/
public static IndexEdge generateScopeToTarget( final Edge edge ) {
- return new IndexEdgeImpl( edge.getTargetNode(), edge.getType(), SearchEdge.NodeType.TARGET, edge.getTimestamp() );
+ return new IndexEdgeImpl( edge.getTargetNode(), edge.getType(), SearchEdge.NodeType.TARGET,
+ edge.getTimestamp() );
}
@@ -120,13 +120,7 @@ public class CpNamingUtils {
/**
- *
* TODO move sourceId to ApplicationScope
- *
- * @param sourceId
- * @param collectionName
- * @param entityId
- * @return
*/
public static Edge createCollectionEdge( final Id sourceId, final String collectionName, final Id entityId ) {
final String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
@@ -205,7 +199,7 @@ public class CpNamingUtils {
public static ApplicationScope getApplicationScope( UUID applicationId ) {
// We can always generate a scope, it doesn't matter if the application exists yet or not.
- return new ApplicationScopeImpl( generateApplicationId( applicationId ) );
+ return new ApplicationScopeImpl( generateApplicationId( applicationId ) );
}
@@ -221,14 +215,12 @@ public class CpNamingUtils {
/**
* Generate an application scope for the management application
- * @return
*/
- public static Id getManagementApplicationId(){
- return generateApplicationId( MANAGEMENT_APPLICATION_ID );
+ public static Id getManagementApplicationId() {
+ return generateApplicationId( MANAGEMENT_APPLICATION_ID );
}
-
/**
* Get the map scope for the applicationId to store entity uuid to type mapping
*/
@@ -268,13 +260,19 @@ public class CpNamingUtils {
}
- private static String getCollectionScopeNameFromCollectionName( String name ) {
+ /**
+ * Only use when searching graph. When searching ES use the <see>createCollectionSearchEdge</see>
+ */
+ public static String getCollectionScopeNameFromCollectionName( String name ) {
String csn = EDGE_COLL_SUFFIX + name;
return csn.toLowerCase();
}
- private static String getConnectionScopeName( String connectionType ) {
+ /**
+ * Only use when searching graph. When searching ES use the <see>createConnectionSearchByEdge</see>
+ */
+ public static String getConnectionScopeName( String connectionType ) {
String csn = EDGE_CONN_SUFFIX + connectionType;
return csn.toLowerCase();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 1c28e6a..930b98e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@ -500,6 +500,9 @@ public class Query {
}
public String getOffsetCursor() {
+
+ //TODO refactor cursor logic for encapsulation at level using it
+
String cursor = "";
if(offset.isPresent()){
ByteBuffer buffer = INTEGER_SERIALIZER.toByteBuffer(offset.get());
@@ -871,7 +874,7 @@ public class Query {
* Return true if no query is present and we should perform a graph search
* @return
*/
- public boolean graphSearch(){
+ public boolean isGraphSearch(){
return ql == null;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
index ca9fb03..df9e094 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgesObservableImpl.java
@@ -65,7 +65,7 @@ public class EdgesObservableImpl implements EdgesObservable {
return gm.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- null ) );
+ Optional.<Edge>absent() ) );
} );
}
@@ -86,7 +86,7 @@ public class EdgesObservableImpl implements EdgesObservable {
return gm.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceNode, edgeType, startTimestamp.or( Long.MIN_VALUE ), SearchByEdgeType.Order.ASCENDING,
- null ) );
+ Optional.<Edge>absent() ) );
} );
}
@@ -123,7 +123,7 @@ public class EdgesObservableImpl implements EdgesObservable {
return gm.loadEdgesToTarget(
new SimpleSearchByEdgeType( targetNode, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
- null ) );
+ Optional.<Edge>absent() ) );
} );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/83174e8b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 7f2ff09..1ee3580 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -89,8 +89,6 @@ public interface IndexFig extends GuicyFig {
String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
- String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
-
@Default( "127.0.0.1" )
@Key( ELASTICSEARCH_HOSTS )
[17/20] incubator-usergrid git commit: Refactor complete. Now need to
test
Posted by to...@apache.org.
Refactor complete. Now need to test
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/9dc65dc8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/9dc65dc8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/9dc65dc8
Branch: refs/heads/two-dot-o-dev
Commit: 9dc65dc8c3176d28d61119c0c434b47b7fc08254
Parents: 0523dc0
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 18:32:36 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 18:32:36 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 163 +++++++++----------
.../pipeline/read/FilterFactory.java | 9 +-
.../pipeline/read/ReadPipelineBuilder.java | 29 ++--
.../pipeline/read/ReadPipelineBuilderImpl.java | 69 +++++---
.../results/AbstractGraphQueryExecutor.java | 133 ---------------
.../results/CollectionGraphQueryExecutor.java | 60 -------
.../results/ConnectionGraphQueryExecutor.java | 58 -------
7 files changed, 139 insertions(+), 382 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9dc65dc8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index d8f9446..af926bf 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -31,12 +31,9 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.index.AsyncIndexService;
-import org.apache.usergrid.corepersistence.results.CollectionGraphQueryExecutor;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.CollectionResultsLoaderFactoryImpl;
-import org.apache.usergrid.corepersistence.results.ConnectionGraphQueryExecutor;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ElasticSearchQueryExecutor;
-import org.apache.usergrid.corepersistence.results.QueryExecutor;
+import org.apache.usergrid.corepersistence.pipeline.PipelineBuilderFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
+import org.apache.usergrid.corepersistence.results.ObservableQueryExecutor;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.ConnectedEntityRef;
@@ -114,6 +111,7 @@ public class CpRelationManager implements RelationManager {
private ManagerCache managerCache;
private EntityCollectionManagerFactory entityCollectionManagerFactory;
private GraphManagerFactory graphManagerFactory;
+ private PipelineBuilderFactory pipelineBuilderFactory;
private EntityManager em;
@@ -131,8 +129,10 @@ public class CpRelationManager implements RelationManager {
private Timer updateCollectionTimer;
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final
- EntityCollectionManagerFactory entityCollectionManagerFactory, final GraphManagerFactory graphManagerFactory, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
+ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final GraphManagerFactory graphManagerFactory, final AsyncIndexService indexService,
+ final EntityManager em, final UUID applicationId, final EntityRef headEntity ) {
Assert.notNull( em, "Entity manager cannot be null" );
@@ -158,8 +158,8 @@ public class CpRelationManager implements RelationManager {
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading head entity {}:{} from app {}", new Object[] {
- headEntity.getType(), headEntity.getUuid(), applicationScope
- } );
+ headEntity.getType(), headEntity.getUuid(), applicationScope
+ } );
}
Id entityId = new SimpleId( headEntity.getUuid(), headEntity.getType() );
@@ -171,24 +171,22 @@ public class CpRelationManager implements RelationManager {
.format( "cpHeadEntity cannot be null for entity id %s, app id %s", entityId.getUuid(), applicationId ) );
this.indexService = indexService;
-
-
}
@Override
public Set<String> getCollectionIndexes( String collectionName ) throws Exception {
- GraphManager gm = managerCache.getGraphManager( applicationScope );
+ GraphManager gm = managerCache.getGraphManager( applicationScope );
String edgeTypePrefix = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
logger.debug( "getCollectionIndexes(): Searching for edge type prefix {} to target {}:{}", new Object[] {
- edgeTypePrefix, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid()
- } );
+ edgeTypePrefix, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid()
+ } );
Observable<Set<String>> types =
- gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeTypePrefix, null ) ).collect(
- () -> new HashSet<>(), ( set, type ) -> set.add( type ) );
+ gm.getEdgeTypesFromSource( new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeTypePrefix, null ) )
+ .collect( () -> new HashSet<>(), ( set, type ) -> set.add( type ) );
return types.toBlocking().last();
@@ -238,7 +236,7 @@ public class CpRelationManager implements RelationManager {
public Observable<Edge> call( final String edgeType ) {
return gm.loadEdgesToTarget(
new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
} );
@@ -249,22 +247,20 @@ public class CpRelationManager implements RelationManager {
return edges.collect( () -> new LinkedHashMap<EntityRef, Set<String>>(), ( entityRefSetMap, edge ) -> {
- if ( fromEntityType != null && !fromEntityType.equals( edge.getSourceNode().getType() ) ) {
- logger.debug( "Ignoring edge from entity type {}", edge.getSourceNode().getType() );
- return;
- }
+ if ( fromEntityType != null && !fromEntityType.equals( edge.getSourceNode().getType() ) ) {
+ logger.debug( "Ignoring edge from entity type {}", edge.getSourceNode().getType() );
+ return;
+ }
- final EntityRef eref =
- new SimpleEntityRef( edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
+ final EntityRef eref =
+ new SimpleEntityRef( edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
- String name = getNameFromEdgeType( edge.getType() );
- addMapSet( entityRefSetMap, eref, name );
- } ).toBlocking().last();
+ String name = getNameFromEdgeType( edge.getType() );
+ addMapSet( entityRefSetMap, eref, name );
+ } ).toBlocking().last();
}
-
-
@Override
public boolean isConnectionMember( String connectionType, EntityRef entity ) throws Exception {
@@ -273,8 +269,8 @@ public class CpRelationManager implements RelationManager {
String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType( connectionType );
logger.debug( "isConnectionMember(): Checking for edge type {} from {}:{} to {}:{}", new Object[] {
- edgeType, headEntity.getType(), headEntity.getUuid(), entity.getType(), entity.getUuid()
- } );
+ edgeType, headEntity.getType(), headEntity.getUuid(), entity.getType(), entity.getUuid()
+ } );
GraphManager gm = managerCache.getGraphManager( applicationScope );
Observable<Edge> edges = gm.loadEdgeVersions(
@@ -294,8 +290,8 @@ public class CpRelationManager implements RelationManager {
String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collName );
logger.debug( "isCollectionMember(): Checking for edge type {} from {}:{} to {}:{}", new Object[] {
- edgeType, headEntity.getType(), headEntity.getUuid(), entity.getType(), entity.getUuid()
- } );
+ edgeType, headEntity.getType(), headEntity.getUuid(), entity.getType(), entity.getUuid()
+ } );
GraphManager gm = managerCache.getGraphManager( applicationScope );
Observable<Edge> edges = gm.loadEdgeVersions(
@@ -306,7 +302,6 @@ public class CpRelationManager implements RelationManager {
}
-
@Override
public Set<String> getCollections() throws Exception {
@@ -408,8 +403,8 @@ public class CpRelationManager implements RelationManager {
if ( logger.isDebugEnabled() ) {
logger.debug( "Loaded member entity {}:{} from app {}\n " + " data {}", new Object[] {
- itemRef.getType(), itemRef.getUuid(), applicationScope, CpEntityMapUtils.toMap( memberEntity )
- } );
+ itemRef.getType(), itemRef.getUuid(), applicationScope, CpEntityMapUtils.toMap( memberEntity )
+ } );
}
@@ -419,14 +414,13 @@ public class CpRelationManager implements RelationManager {
gm.writeEdge( edge ).toBlocking().last();
-
//perform indexing
if ( logger.isDebugEnabled() ) {
logger.debug( "Wrote edge {}", edge );
}
- indexService.queueEntityIndexUpdate( applicationScope, memberEntity);
+ indexService.queueEntityIndexUpdate( applicationScope, memberEntity );
if ( logger.isDebugEnabled() ) {
@@ -492,7 +486,7 @@ public class CpRelationManager implements RelationManager {
addToCollection( collName, itemEntity );
if ( collection != null && collection.getLinkedCollection() != null ) {
- Id itemEntityId = new SimpleId( itemEntity.getUuid(),itemEntity.getType() );
+ Id itemEntityId = new SimpleId( itemEntity.getUuid(), itemEntity.getType() );
final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collName, itemEntityId );
GraphManager gm = managerCache.getGraphManager( applicationScope );
@@ -528,8 +522,8 @@ public class CpRelationManager implements RelationManager {
if ( logger.isDebugEnabled() ) {
logger.debug( "Loading entity to remove from collection " + "{}:{} from app {}\n", new Object[] {
- itemRef.getType(), itemRef.getUuid(), applicationScope
- } );
+ itemRef.getType(), itemRef.getUuid(), applicationScope
+ } );
}
Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
@@ -636,38 +630,30 @@ public class CpRelationManager implements RelationManager {
}
-
query.setEntityType( collection.getType() );
query = adjustQuery( query );
+ final ReadPipelineBuilder readPipelineBuilder =
+ pipelineBuilderFactory.createReadPipelineBuilder( applicationScope );
- if ( query.isGraphSearch() ) {
- QueryExecutor executor =
- new CollectionGraphQueryExecutor( entityCollectionManagerFactory, graphManagerFactory, applicationScope,
- headEntity, query.getOffsetCursor(), collName, query.getLimit() );
-
- return executor.next();
- }
+ //set our fields applicable to both operations
+ readPipelineBuilder.withCursor( query.getOffsetCursor() );
+ readPipelineBuilder.withLimit( query.getLimit() );
+ //TODO, this should be removed when the CP relation manager is removed
+ readPipelineBuilder.setStartId( cpHeadEntity.getId() );
-
- final SearchEdge searchEdge = createCollectionSearchEdge( cpHeadEntity.getId(), collName );
-
- final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
-
- final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
-
- logger.debug( "Searching scope {}", searchEdge );
-
- final CollectionResultsLoaderFactoryImpl resultsLoaderFactory =
- new CollectionResultsLoaderFactoryImpl( managerCache );
+ if ( query.isGraphSearch() ) {
+ readPipelineBuilder.getCollection( collName );
+ }
+ else {
+ readPipelineBuilder.getCollectionWithQuery( collName, query.getQl().get() );
+ }
- //execute the query and return our next result
- final QueryExecutor executor =
- new ElasticSearchQueryExecutor( resultsLoaderFactory, ei, applicationScope, searchEdge, types, query );
+ final Observable<Results> resultsObservable = readPipelineBuilder.build();
- return executor.next();
+ return new ObservableQueryExecutor( resultsObservable ).next();
}
@@ -689,10 +675,10 @@ public class CpRelationManager implements RelationManager {
if ( logger.isDebugEnabled() ) {
logger.debug( "createConnection(): " + "Indexing connection type '{}'\n from source {}:{}]\n"
- + " to target {}:{}\n app {}", new Object[] {
- connectionType, headEntity.getType(), headEntity.getUuid(), connectedEntityRef.getType(),
- connectedEntityRef.getUuid(), applicationScope
- } );
+ + " to target {}:{}\n app {}", new Object[] {
+ connectionType, headEntity.getType(), headEntity.getUuid(), connectedEntityRef.getType(),
+ connectedEntityRef.getUuid(), applicationScope
+ } );
}
Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() );
@@ -778,9 +764,9 @@ public class CpRelationManager implements RelationManager {
if ( logger.isDebugEnabled() ) {
logger.debug( "Deleting connection '{}' from source {}:{} \n to target {}:{}", new Object[] {
- connectionType, connectingEntityRef.getType(), connectingEntityRef.getUuid(),
- connectedEntityRef.getType(), connectedEntityRef.getUuid()
- } );
+ connectionType, connectingEntityRef.getType(), connectingEntityRef.getUuid(),
+ connectedEntityRef.getType(), connectedEntityRef.getUuid()
+ } );
}
Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() );
@@ -909,34 +895,36 @@ public class CpRelationManager implements RelationManager {
headEntity = em.validate( headEntity );
- final SearchEdge indexScope = createConnectionSearchEdge( cpHeadEntity.getId(), connection );
- final SearchTypes searchTypes = SearchTypes.fromNullableTypes( query.getEntityType() );
+ query = adjustQuery( query );
- ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final String entityType = query.getEntityType();
- logger.debug( "Searching {}", indexScope );
- query = adjustQuery( query );
+ final ReadPipelineBuilder readPipelineBuilder =
+ pipelineBuilderFactory.createReadPipelineBuilder( applicationScope );
+ //set our fields applicable to both operations
+ readPipelineBuilder.withCursor( query.getOffsetCursor() );
+ readPipelineBuilder.withLimit( query.getLimit() );
+ //TODO, this should be removed when the CP relation manager is removed
+ readPipelineBuilder.setStartId( cpHeadEntity.getId() );
if ( query.isGraphSearch() ) {
- QueryExecutor executor =
- new ConnectionGraphQueryExecutor( entityCollectionManagerFactory, graphManagerFactory, applicationScope,
- headEntity, query.getOffsetCursor(), connection, query.getLimit() );
-
- return executor.next();
+ readPipelineBuilder.getConnection( connection );
+ }
+ else if ( entityType != null ) {
+ readPipelineBuilder.connectionWithQuery( connection, query.getQl().get(), entityType );
+ }
+ else {
+ readPipelineBuilder.connectionWithQuery( connection, query.getQl().get() );
}
- final ConnectionResultsLoaderFactoryImpl resultsLoaderFactory =
- new ConnectionResultsLoaderFactoryImpl( managerCache, headEntity, connection );
- final QueryExecutor executor =
- new ElasticSearchQueryExecutor( resultsLoaderFactory, ei, applicationScope, indexScope, searchTypes,
- query );
+ final Observable<Results> resultsObservable = readPipelineBuilder.build();
- return executor.next();
+ return new ObservableQueryExecutor( resultsObservable ).next();
}
@@ -1007,7 +995,6 @@ public class CpRelationManager implements RelationManager {
}
-
/** side effect: converts headEntity into an Entity if it is an EntityRef! */
private Entity getHeadEntity() throws Exception {
Entity entity = null;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9dc65dc8/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
index 525678c..8f9776e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
@@ -80,22 +80,19 @@ public interface FilterFactory {
/**
* Generate a new instance of the command with the specified parameters
*/
- QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector( final String collectionName, final String query,
- final Optional<String> cursor, final int limit );
+ QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector( final String collectionName, final String query);
/**
* Generate a new instance of the command with the specified parameters
*/
- QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName,final String query,
- final Optional<String> cursor, final int limit );
+ QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName,final String query);
/**
* Generate a new instance of the command with the specified parameters
*/
- QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName, final String connectionEntityType, final String query,
- final Optional<String> cursor, final int limit );
+ QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName, final String connectionEntityType, final String query );
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9dc65dc8/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
index 13aa3eb..d40cb12 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
@@ -20,13 +20,9 @@
package org.apache.usergrid.corepersistence.pipeline.read;
-import java.util.UUID;
-
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.model.entity.Id;
-import com.google.common.base.Optional;
-
import rx.Observable;
@@ -37,6 +33,20 @@ import rx.Observable;
*/
public interface ReadPipelineBuilder {
+
+ /**
+ * Set the cursor
+ * @param cursor
+ */
+ ReadPipelineBuilder withCursor(final String cursor);
+
+ /**
+ * Set the limit of our page sizes
+ * @param limit
+ * @return
+ */
+ ReadPipelineBuilder withLimit(final int limit);
+
/**
* An operation to bridge 2.0-> 1.0. Should be removed when everyone uses the pipeline
* @param id
@@ -58,8 +68,7 @@ public interface ReadPipelineBuilder {
/**
* Get all entities with a query
*/
- ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query, final int limit,
- final Optional<String> cursor );
+ ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query);
/**
* Get an entity via the connection name and entity Id
@@ -79,15 +88,13 @@ public interface ReadPipelineBuilder {
/**
* Get all entities in a connection with a query
*/
- ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query, final int limit,
- final Optional<String> cursor );
+ ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query );
/**
* Get all entities in a connection with a query
*/
- ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType, final String query, final int limit,
- final Optional<String> cursor );
+ ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType, final String query);
@@ -98,5 +105,5 @@ public interface ReadPipelineBuilder {
* Execute final construction of the pipeline and return the results
* @return
*/
- Observable<Results> execute();
+ Observable<Results> build();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9dc65dc8/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
index a44f4b8..7319f21 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
@@ -20,11 +20,7 @@
package org.apache.usergrid.corepersistence.pipeline.read;
-import java.util.Collections;
-import java.util.UUID;
-
import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
-import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -48,17 +44,41 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
private final DataPipeline pipeline;
/**
- * Our pointer to our collect filter. Set or cleared with each operation that's performed so the correct
- * results are rendered
+ * Our pointer to our collect filter. Set or cleared with each operation that's performed so the correct results are
+ * rendered
*/
private CollectorFilter<Results> collectorFilter;
+ private Optional<String> cursor;
+ private Optional<Integer> limit;
+
@Inject
public ReadPipelineBuilderImpl( final FilterFactory filterFactory,
@Assisted final ApplicationScope applicationScope ) {
this.filterFactory = filterFactory;
this.pipeline = new DataPipeline( applicationScope );
+ this.cursor = Optional.absent();
+ //set the default limit
+ this.limit = Optional.absent();
+ }
+
+
+ @Override
+ public ReadPipelineBuilder withCursor( final String cursor ) {
+ this.cursor = Optional.fromNullable( cursor );
+ pipeline.setCursor( this.cursor );
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder withLimit( final int limit ) {
+ Preconditions.checkArgument( limit > 0, "You must set the limit > 0" );
+ this. limit = Optional.of( limit );
+ //set the default value
+ pipeline.setLimit( this.limit.or( 10 ) );
+ return this;
}
@@ -97,11 +117,10 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
- public ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query, final int limit,
- final Optional<String> cursor ) {
+ public ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query ) {
//TODO, this should really be 2 a TraverseFilter with an entityLoad collector
- collectorFilter = filterFactory.queryCollectionElasticSearchCollector( collectionName, query, cursor, limit );
+ collectorFilter = filterFactory.queryCollectionElasticSearchCollector( collectionName, query );
return this;
}
@@ -129,25 +148,23 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
public ReadPipelineBuilder getConnection( final String connectionName, final String entityType ) {
pipeline.withTraverseCommand( filterFactory.readGraphConnectionCommand( connectionName, entityType ) );
- setEntityLoaderFilter();
+ setEntityLoaderFilter();
return this;
}
+
/**
*
* @param connectionName
* @param query
- * @param limit
- * @param cursor
* @return
*/
@Override
- public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query, final int limit,
- final Optional<String> cursor ) {
+ public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query ) {
- //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
- collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, query, cursor, limit );
+ //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+ collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, query );
return this;
}
@@ -155,26 +172,26 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType,
- final String query, final int limit,
- final Optional<String> cursor ) {
+ final String query ) {
- //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
- collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, entityType, query, cursor, limit );
+ //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+ collectorFilter =
+ filterFactory.queryConnectionElasticSearchCollector( connectionName, entityType, query);
return this;
}
-
-
@Override
- public Observable<Results> execute() {
- Preconditions.checkNotNull(collectorFilter, "You have not specified an operation that creates a collection filter. This is required for loading results");
+ public Observable<Results> build() {
+ Preconditions.checkNotNull( collectorFilter,
+ "You have not specified an operation that creates a collection filter. This is required for loading "
+ + "results" );
return pipeline.build( collectorFilter );
}
- private void setEntityLoaderFilter(){
+
+ private void setEntityLoaderFilter() {
collectorFilter = filterFactory.entityLoadCollector();
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9dc65dc8/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
deleted file mode 100644
index 663c15c..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
-import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.entity.SimpleId;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * This class is a nasty hack to bridge 2.0 observables into 1.0 iterators DO NOT use this as a model for moving
- * forward, pandas will die.
- */
-public abstract class AbstractGraphQueryExecutor implements QueryExecutor {
-
-
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private final ApplicationScope applicationScope;
- private final Id sourceId;
- private final int limit;
-
- private final Optional<String> requestCursor;
-
- private Iterator<Results> observableIterator;
-
-
- public AbstractGraphQueryExecutor( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final ApplicationScope applicationScope, final EntityRef source,
- final String cursor, final int limit ) {
-
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.applicationScope = applicationScope;
- this.limit = limit;
- this.sourceId = new SimpleId( source.getUuid(), source.getType() );
-
- this.requestCursor = Optional.fromNullable( cursor );
- }
-
-
- @Override
- public Iterator<Results> iterator() {
- return this;
- }
-
-
- @Override
- public boolean hasNext() {
-
- //hasn't been set up yet, run through our first setup
- if ( observableIterator == null ) {
- //assign them to an iterator. this now uses an internal buffer with backpressure, so we won't load all
- // results
- //set up our command builder
- final DataPipeline dataPipeline = new DataPipeline( applicationScope, sourceId, requestCursor, limit );
-
-
- addTraverseCommand( dataPipeline );
-
- //construct our results to be observed later. This is a cold observable
- final Observable<Results> resultsObservable =
- dataPipeline.build( new EntityLoadCollectorFilter( entityCollectionManagerFactory, applicationScope ) );
-
- this.observableIterator = resultsObservable.toBlocking().getIterator();
-
- if(!observableIterator.hasNext()){
- //no results, generate an empty one
- this.observableIterator = Collections.singleton(new Results()).iterator();
- }
- }
-
-
- //see if our current results are not null
- return observableIterator.hasNext();
- }
-
-
- @Override
- public Results next() {
- if ( !hasNext() ) {
- throw new NoSuchElementException( "No more results present" );
- }
-
- final Results results = observableIterator.next();
-
- //ugly and tight coupling, but we don't have a choice until we finish some refactoring
- results.setQueryExecutor( this );
-
- return results;
- }
-
-
- @Override
- public void remove() {
- throw new RuntimeException( "Remove not implemented!!" );
- }
-
-
- /**
- * Add the traverse command to the graph
- */
- protected abstract void addTraverseCommand( final DataPipeline dataPipeline );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9dc65dc8/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
deleted file mode 100644
index b506c71..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.common.base.Preconditions;
-
-
-/**
- * Create The collection graph query executor
- */
-public class CollectionGraphQueryExecutor extends AbstractGraphQueryExecutor {
-
- private final GraphManagerFactory graphManagerFactory;
- private final String collectionName;
-
-
- public CollectionGraphQueryExecutor( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final GraphManagerFactory graphManagerFactory,
- final ApplicationScope applicationScope, final EntityRef source,
- final String cursor, final String collectionName, final int limit ) {
-
- super( entityCollectionManagerFactory, applicationScope, source, cursor, limit );
- this.graphManagerFactory = graphManagerFactory;
-
- Preconditions.checkNotNull( collectionName, "collectionName is required on the query" );
- this.collectionName = collectionName;
- }
-
-
- @Override
- protected void addTraverseCommand( final DataPipeline dataPipeline ) {
- //set the traverse command from the source Id to the connect name
- dataPipeline.withTraverseCommand( new ReadGraphCollectionFilter( graphManagerFactory, collectionName ) );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/9dc65dc8/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
deleted file mode 100644
index 3dc5cd6..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.common.base.Preconditions;
-
-
-public class ConnectionGraphQueryExecutor extends AbstractGraphQueryExecutor {
-
- private final GraphManagerFactory graphManagerFactory;
- private final String connectionName;
-
-
- public ConnectionGraphQueryExecutor( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final GraphManagerFactory graphManagerFactory,
- final ApplicationScope applicationScope, final EntityRef source,
- final String cursor, final String connectionType, final int limit) {
-
- super( entityCollectionManagerFactory, applicationScope, source, cursor, limit );
- this.graphManagerFactory = graphManagerFactory;
-
- Preconditions.checkNotNull(connectionType, "connectionType is required on the query" );
- this.connectionName = connectionType;
- }
-
-
-
- @Override
- protected void addTraverseCommand( final DataPipeline dataPipeline ) {
- //set the traverse command from the source Id to the connect name
- dataPipeline.withTraverseCommand( new ReadGraphConnectionFilter( graphManagerFactory, connectionName ) );
- }
-}
[11/20] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into USERGRID-593
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-593
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/60617d2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/60617d2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/60617d2a
Branch: refs/heads/two-dot-o-dev
Commit: 60617d2a7ae3309a850959e62228826ff789b878
Parents: 83174e8 11e40ce
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 10:33:50 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 10:33:50 2015 -0600
----------------------------------------------------------------------
.../persistence/query/CollectionIoHelper.java | 1 -
.../persistence/query/IteratingQueryIT.java | 21 +++++------
.../index/impl/EsEntityIndexImpl.java | 12 +++----
.../index/impl/EsIndexBufferConsumerImpl.java | 18 ++++++++--
.../index/impl/IndexRefreshCommandImpl.java | 2 +-
.../persistence/index/impl/EntityIndexTest.java | 37 +++++++-------------
6 files changed, 43 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
[08/20] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into USERGRID-593
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
index 0000000,e5fbac9..1c28e6a
mode 000000,100644..100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/Query.java
@@@ -1,0 -1,921 +1,912 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.usergrid.persistence;
+
+
+ import java.io.IOException;
+ import java.io.Serializable;
+ import java.io.UnsupportedEncodingException;
+ import java.net.URLDecoder;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.Map.Entry;
+ import java.util.UUID;
+
+ import com.netflix.astyanax.serializers.IntegerSerializer;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ import org.apache.commons.codec.binary.Base64;
+ import org.apache.commons.lang.StringUtils;
+
+ import org.apache.usergrid.persistence.index.SelectFieldMapping;
+ import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
+ import org.apache.usergrid.persistence.index.query.CounterResolution;
+ import org.apache.usergrid.persistence.index.query.Identifier;
+ import org.apache.usergrid.persistence.index.query.tree.Operand;
+ import org.apache.usergrid.persistence.index.utils.ClassUtils;
+ import org.apache.usergrid.persistence.index.utils.ConversionUtils;
+ import org.apache.usergrid.persistence.index.utils.ListUtils;
+ import org.apache.usergrid.persistence.index.utils.MapUtils;
+
+ import com.fasterxml.jackson.annotation.JsonIgnore;
+ import com.fasterxml.jackson.databind.ObjectMapper;
+ import com.google.common.base.Optional;
+
+
+ public class Query {
+ private static final Logger logger = LoggerFactory.getLogger( Query.class );
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+
+ public enum Level {
+ IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
+ }
+
+ public static final int DEFAULT_LIMIT = 10;
+
+ public static final int MAX_LIMIT = 1000;
+
+ public static final String PROPERTY_UUID = "uuid";
+
+ private String type;
+ private Operand rootOperand;
+ private UUID startResult;
+ private Optional<Integer> offset;
+ private int limit = 0;
+
+ private boolean mergeSelectResults = false;
+ private Level level = Level.ALL_PROPERTIES;
+ private String connectionType;
+ private List<String> permissions;
+ private boolean reversed;
+ private boolean reversedSet = false;
+ private Long startTime;
+ private Long finishTime;
+ private boolean pad;
+ private CounterResolution resolution = CounterResolution.ALL;
+ private List<Identifier> identifiers;
+ private List<CounterFilterPredicate> counterFilters;
+ private String collection;
+ private String ql;
+ private Collection<SelectFieldMapping> selectFields;
+
+
+ private static ObjectMapper mapper = new ObjectMapper();
+
+
+ List<Operand> filterClauses = new ArrayList<Operand>();
+
+ public Query() {
+ offset = Optional.absent();
+ }
+
+
+ /**
+ * Creates a deep copy of a query from another query
+ * @param q
+ */
+ public Query( Query q ) {
+ if ( q == null ) {
+ return;
+ }
+
+ ql = q.ql;
+ type = q.type;
+ startResult = q.startResult;
+ offset = q.offset;
+ limit = q.limit;
+ mergeSelectResults = q.mergeSelectResults;
+ //level = q.level;
+ connectionType = q.connectionType;
+ permissions = q.permissions != null ? new ArrayList<>( q.permissions ) : null;
+ reversed = q.reversed;
+ reversedSet = q.reversedSet;
+ startTime = q.startTime;
+ finishTime = q.finishTime;
+ resolution = q.resolution;
+ pad = q.pad;
+ rootOperand = q.rootOperand;
+ identifiers = q.identifiers != null
+ ? new ArrayList<>( q.identifiers ) : null;
+ counterFilters = q.counterFilters != null
+ ? new ArrayList<>( q.counterFilters ) : null;
+ collection = q.collection;
+ level = q.level;
+
+ }
+
+
+
+ /**
+ * Create a query instance from the QL. If the string is null, return an empty query
+ * @param ql
+ * @return
+ */
+ public static Query fromQLNullSafe(final String ql){
+ final Query query = fromQL(ql);
+
+ if(query != null){
+ return query;
+ }
+
+ return new Query();
+ }
+
+ public static Query fromQL( String ql ) throws QueryParseException {
+ if ( StringUtils.isEmpty(ql) ) {
+ return null;
+ }
+
+ Query query = new Query( );
+ query.setQl( ql );
+
+ return query;
+ }
+
+
+ /**
+ * Create a query from a property equals
+ * @param propertyName
+ * @param value
+ * @return
+ */
+ public static Query fromEquals(final String propertyName, final String value){
+ return fromQL( propertyName + " = '" + value + "'" );
+ }
+
+
+ private static Query newQueryIfNull( Query query ) {
+ if ( query == null ) {
+ query = new Query();
+ }
+ return query;
+ }
+
+
- public static Query fromJsonString( String json ) throws QueryParseException {
-
- Object o;
- try {
- o = mapper.readValue( json, Object.class );
- } catch (IOException ex) {
- throw new QueryParseException("Error parsing JSON query string " + json, ex);
- }
-
- if ( o instanceof Map ) {
- @SuppressWarnings({ "unchecked", "rawtypes" }) Map<String, List<String>> params =
- ClassUtils.cast( MapUtils.toMapList( ( Map ) o ) );
- return fromQueryParams( params );
- }
- return null;
- }
-
+
+ public static Query fromQueryParams( Map<String, List<String>> params )
+ throws QueryParseException {
+ Query q = null;
+ CounterResolution resolution = null;
+ List<Identifier> identifiers = null;
+ List<CounterFilterPredicate> counterFilters = null;
+
+ String ql = QueryUtils.queryStrFrom(params);
+ String type = ListUtils.first(params.get("type"));
+ Boolean reversed = ListUtils.firstBoolean(params.get("reversed"));
+ String connection = ListUtils.first(params.get("connectionType"));
+ UUID start = ListUtils.firstUuid(params.get("start"));
+ String cursor = ListUtils.first(params.get("cursor"));
+
+ Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
+ List<String> permissions = params.get( "permission" );
+ Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
+ Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
+
+ List<String> l = params.get( "resolution" );
+ if ( !ListUtils.isEmpty( l ) ) {
+ resolution = CounterResolution.fromString( l.get( 0 ) );
+ }
+
+ l = params.get( "counter" );
+
+ if ( !ListUtils.isEmpty( l ) ) {
+ counterFilters = CounterFilterPredicate.fromList( l );
+ }
+
+ Boolean pad = ListUtils.firstBoolean( params.get( "pad" ) );
+
+ for ( Entry<String, List<String>> param : params.entrySet() ) {
+ Identifier identifier = Identifier.from( param.getKey() );
+ if ( ( param.getValue() == null ) || ( param.getValue().size() == 0 ) || identifier.isUUID() ) {
+ if ( identifier != null ) {
+ if ( identifiers == null ) {
+ identifiers = new ArrayList<Identifier>();
+ }
+ identifiers.add( identifier );
+ }
+ }
+ }
+
+ if ( ql != null ) {
+ q = Query.fromQL( decode( ql ) );
+ }
+
+
+ if ( type != null ) {
+ q = newQueryIfNull( q );
+ q.setEntityType( type );
+ }
+
+ if ( connection != null ) {
+ q = newQueryIfNull( q );
+ q.setConnectionType( connection );
+ }
+
+ if ( permissions != null ) {
+ q = newQueryIfNull( q );
+ q.setPermissions( permissions );
+ }
+
+ if ( start != null ) {
+ q = newQueryIfNull( q );
+ q.setStartResult( start );
+ }
+
+ if ( cursor != null) {
+ q = newQueryIfNull( q );
+ q.setOffsetFromCursor(cursor);
+ }else{
+ q.offset = Optional.absent();
+ }
+
+ if ( limit != null ) {
+ q = newQueryIfNull( q );
+ q.setLimit( limit );
+ }
+
+ if ( startTime != null ) {
+ q = newQueryIfNull( q );
+ q.setStartTime( startTime );
+ }
+
+ if ( finishTime != null ) {
+ q = newQueryIfNull( q );
+ q.setFinishTime( finishTime );
+ }
+
+ if ( resolution != null ) {
+ q = newQueryIfNull( q );
+ q.setResolution( resolution );
+ }
+
+ if ( counterFilters != null ) {
+ q = newQueryIfNull( q );
+ q.setCounterFilters( counterFilters );
+ }
+
+ if ( pad != null ) {
+ q = newQueryIfNull( q );
+ q.setPad( pad );
+ }
+
+ if ( identifiers != null ) {
+ q = newQueryIfNull( q );
+ q.setIdentifiers( identifiers );
+ }
+
+ if ( reversed != null ) {
+ q = newQueryIfNull( q );
+ q.setReversed( reversed );
+ }
+
+ return q;
+ }
+
+
+ public static Query fromUUID( UUID uuid ) {
+ Query q = new Query();
+ q.addIdentifier( Identifier.fromUUID(uuid) );
+ return q;
+ }
+
+
+ public static Query fromIdentifier( Object id ) {
+ Query q = new Query();
+ q.addIdentifier( Identifier.from(id) );
+ return q;
+ }
+
+
+ public boolean hasQueryPredicates() {
+ return rootOperand != null;
+ }
+
+
+ /**
+ * Return true if the query generated select subjects
+ * @return
+ */
+ public boolean hasSelectSubjects() {
+ return this.selectFields != null || this.selectFields.size() > 0;
+ }
+
+
+ /**
+ * Set the select subjects from our query results
+ */
+ public void setSelectSubjects( final Collection<SelectFieldMapping> selectFields ) {
+ this.selectFields = selectFields;
+ }
+
+
+ /**
+ * Get the select assignments from our resetus if they were set
+ */
+ public Collection<SelectFieldMapping> getSelectAssignments() {
+ return this.selectFields;
+ }
+
+
+ public boolean containsNameOrEmailIdentifiersOnly() {
+ if ( hasQueryPredicates() ) {
+ return false;
+ }
+ if ( ( identifiers == null ) || identifiers.isEmpty() ) {
+ return false;
+ }
+ for ( Identifier identifier : identifiers ) {
+ if ( !identifier.isEmail() && !identifier.isName() ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ @JsonIgnore
+ public String getSingleNameOrEmailIdentifier() {
+ if ( !containsSingleNameOrEmailIdentifier() ) {
+ return null;
+ }
+ return ( identifiers.get( 0 ).toString() );
+ }
+
+
+ public boolean containsSingleNameOrEmailIdentifier() {
+ return containsNameOrEmailIdentifiersOnly() && ( identifiers.size() == 1 );
+ }
+
+
+ @JsonIgnore
+ public Identifier getSingleIdentifier() {
+ return identifiers != null && identifiers.size() == 1 ? identifiers.get( 0 ) : null;
+ }
+
+
+ public boolean containsSingleUuidIdentifier() {
+ return containsUuidIdentifiersOnly() && ( identifiers.size() == 1 );
+ }
+
+
+ boolean containsUuidIdentifiersOnly() {
+ if ( hasQueryPredicates() ) {
+ return false;
+ }
+ if ( ( identifiers == null ) || identifiers.isEmpty() ) {
+ return false;
+ }
+
+ for ( Identifier identifier : identifiers ) {
+ if ( !identifier.isUUID() ) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+
+ @JsonIgnore
+ public UUID getSingleUuidIdentifier() {
+ if ( !containsSingleUuidIdentifier() ) {
+ return null;
+ }
+ return ( identifiers.get( 0 ).getUUID() );
+ }
+
+
+ @JsonIgnore
+ boolean isIdsOnly() {
+ return false;
+ }
+
+ private void setIdsOnly( boolean idsOnly ) {
+ if ( idsOnly ) {
+ level = Level.IDS;
+ }
+ else if ( isIdsOnly() ) {
+ level = Level.ALL_PROPERTIES;
+ }
+ }
+
+
+ public Level getResultsLevel() {
+ isIdsOnly();
+ return level;
+ }
+
+
+ public void setResultsLevel( Level level ) {
+ setIdsOnly( level == Level.IDS );
+ this.level = level;
+ }
+
+
+ public Query withResultsLevel( Level level ) {
+ setIdsOnly( level == Level.IDS );
+ this.level = level;
+ return this;
+ }
+
+
+ public Query withReversed( boolean reversed ) {
+ setReversed( reversed );
+ return this;
+ }
+
+
+ public String getEntityType() {
+ return type;
+ }
+
+
+ public void setEntityType( String type ) {
+ this.type = type;
+ }
+
+ public List<String> getPermissions() {
+ return permissions;
+ }
+
+
+ public void setPermissions( List<String> permissions ) {
+ this.permissions = permissions;
+ }
+
+
+ public boolean isMergeSelectResults() {
+ return mergeSelectResults;
+ }
+
+
+
+ void setStartResult( UUID startResult ) {
+ this.startResult = startResult;
+ }
+
+
+ public Query withStartResult( UUID startResult ) {
+ this.startResult = startResult;
+ return this;
+ }
+
+
+ public UUID getStartResult() {
+ return startResult;
+ }
+
+
+ public Optional<Integer> getOffset() {
+ return offset;
+ }
+
+ public String getOffsetCursor() {
+ String cursor = "";
+ if(offset.isPresent()){
+ ByteBuffer buffer = INTEGER_SERIALIZER.toByteBuffer(offset.get());
+ cursor = Base64.encodeBase64String(buffer.array());
+ }
+ return cursor;
+ }
+
+ public void setOffsetFromCursor(String cursor) {
+ if(cursor == null || cursor.length() == 0){
+ clearOffset();
+ }else {
+ byte[] bytes = Base64.decodeBase64(cursor);
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
+ Integer number = INTEGER_SERIALIZER.fromByteBuffer(buffer);
+ setOffset(number);
+ }
+ }
+
+ public void clearOffset() {
+ this.offset = Optional.absent();
+ }
+
+ public void setOffset( int offset ) {
+ this.offset = Optional.of(offset);
+ }
+
+
+ public Query withOffset( int offset ) {
+ setOffset(offset);
+ return this;
+ }
+
+ public Query withOffsetFromCursor( String cursor ) {
+ setOffsetFromCursor(cursor);
+ return this;
+ }
+
+
+ public int getLimit() {
+ return getLimit( DEFAULT_LIMIT );
+ }
+
+
+ public int getLimit( int defaultLimit ) {
+ if ( limit <= 0 ) {
+ if ( defaultLimit > 0 ) {
+ return defaultLimit;
+ }
+ else {
+ return DEFAULT_LIMIT;
+ }
+ }
+ return limit;
+ }
+
+
+ public void setLimit( int limit ) {
+
+ // TODO tnine. After users have had time to change their query limits,
+ // this needs to be uncommented and enforced.
+ // if(limit > MAX_LIMIT){
+ // throw new IllegalArgumentException(
+ // String.format("Query limit must be <= to %d", MAX_LIMIT));
+ // }
+
+ if ( limit > MAX_LIMIT ) {
+ limit = MAX_LIMIT;
+ }
+
+ this.limit = limit;
+ }
+
+
+ public Query withLimit( int limit ) {
+ setLimit( limit );
+ return this;
+ }
+
+
+ public boolean isReversed() {
+ return reversed;
+ }
+
+
+ public void setReversed( boolean reversed ) {
+ reversedSet = true;
+ this.reversed = reversed;
+ }
+
+
+ public boolean isReversedSet() {
+ return reversedSet;
+ }
+
+
+ public Long getStartTime() {
+ return startTime;
+ }
+
+
+ public void setStartTime( Long startTime ) {
+ this.startTime = startTime;
+ }
+
+
+ public Long getFinishTime() {
+ return finishTime;
+ }
+
+
+ public void setFinishTime( Long finishTime ) {
+ this.finishTime = finishTime;
+ }
+
+
+ public boolean isPad() {
+ return pad;
+ }
+
+
+ public void setPad( boolean pad ) {
+ this.pad = pad;
+ }
+
+
+ public void setResolution( CounterResolution resolution ) {
+ this.resolution = resolution;
+ }
+
+
+ public CounterResolution getResolution() {
+ return resolution;
+ }
+
+
+ public void addIdentifier( Identifier identifier ) {
+ if ( identifiers == null ) {
+ identifiers = new ArrayList<Identifier>();
+ }
+ identifiers.add( identifier );
+ }
+
+
+ void setIdentifiers( List<Identifier> identifiers ) {
+ this.identifiers = identifiers;
+ }
+
+
+ public List<CounterFilterPredicate> getCounterFilters() {
+ return counterFilters;
+ }
+
+
+ public void addCounterFilter( String counter ) {
+ CounterFilterPredicate p = CounterFilterPredicate.fromString( counter );
+ if ( p == null ) {
+ return;
+ }
+ if ( counterFilters == null ) {
+ counterFilters = new ArrayList<CounterFilterPredicate>();
+ }
+ counterFilters.add( p );
+ }
+
+
+ void setCounterFilters( List<CounterFilterPredicate> counterFilters ) {
+ this.counterFilters = counterFilters;
+ }
+
+
+
+ public static final class CounterFilterPredicate implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final String name;
+ private final Identifier user;
+ private final Identifier group;
+ private final String queue;
+ private final String category;
+
+
+ public CounterFilterPredicate( String name, Identifier user, Identifier group, String queue, String category ) {
+ this.name = name;
+ this.user = user;
+ this.group = group;
+ this.queue = queue;
+ this.category = category;
+ }
+
+
+ public Identifier getUser() {
+ return user;
+ }
+
+
+ public Identifier getGroup() {
+ return group;
+ }
+
+
+ public String getQueue() {
+ return queue;
+ }
+
+
+ public String getCategory() {
+ return category;
+ }
+
+
+ public String getName() {
+ return name;
+ }
+
+
+ public static CounterFilterPredicate fromString( String s ) {
+ Identifier user = null;
+ Identifier group = null;
+ String category = null;
+ String name = null;
+ String[] l = StringUtils.split( s, ':' );
+
+ if ( l.length > 0 ) {
+ if ( !"*".equals( l[0] ) ) {
+ name = l[0];
+ }
+ }
+
+ if ( l.length > 1 ) {
+ if ( !"*".equals( l[1] ) ) {
+ user = Identifier.from( l[1] );
+ }
+ }
+
+ if ( l.length > 2 ) {
+ if ( !"*".equals( l[2] ) ) {
+ group = Identifier.from( l[3] );
+ }
+ }
+
+ if ( l.length > 3 ) {
+ if ( !"*".equals( l[3] ) ) {
+ category = l[3];
+ }
+ }
+
+ if ( ( user == null ) && ( group == null ) && ( category == null ) && ( name == null)) {
+ return null;
+ }
+
+ return new CounterFilterPredicate( name, user, group, null, category );
+ }
+
+
+ public static List<CounterFilterPredicate> fromList( List<String> l ) {
+ if ( ( l == null ) || ( l.size() == 0 ) ) {
+ return null;
+ }
+ List<CounterFilterPredicate> counterFilters = new ArrayList<CounterFilterPredicate>();
+ for ( String s : l ) {
+ CounterFilterPredicate filter = CounterFilterPredicate.fromString( s );
+ if ( filter != null ) {
+ counterFilters.add( filter );
+ }
+ }
+ if ( counterFilters.size() == 0 ) {
+ return null;
+ }
+ return counterFilters;
+ }
+ }
+
+
+ // public List<Object> getSelectionResults( Results rs ) {
+ //
+ // List<Entity> entities = rs.getEntities();
+ // if ( entities == null ) {
+ // return null;
+ // }
+ //
+ // if ( !hasSelectSubjects() ) {
+ // return cast( entities );
+ // }
+ //
+ // List<Object> results = new ArrayList<Object>();
+ //
+ // for ( Entity entity : entities ) {
+ // if ( isMergeSelectResults() ) {
+ // boolean include = false;
+ // Map<String, Object> result = new LinkedHashMap<String, Object>();
+ // Map<String, String> selects = getSelectAssignments();
+ // for ( Map.Entry<String, String> select : selects.entrySet() ) {
+ // Object obj = JsonUtils.select( entity, select.getValue(), false );
+ // if ( obj != null ) {
+ // include = true;
+ // }
+ // result.put( select.getKey(), obj );
+ // }
+ // if ( include ) {
+ // results.add( result );
+ // }
+ // }
+ // else {
+ // boolean include = false;
+ // List<Object> result = new ArrayList<Object>();
+ // Set<String> selects = getSelectSubjects();
+ // for ( String select : selects ) {
+ // Object obj = JsonUtils.select( entity, select );
+ // if ( obj != null ) {
+ // include = true;
+ // }
+ // result.add( obj );
+ // }
+ // if ( include ) {
+ // results.add( result );
+ // }
+ // }
+ // }
+ //
+ // if ( results.size() == 0 ) {
+ // return null;
+ // }
+ //
+ // return results;
+ // }
+
+
+ // public Object getSelectionResult( Results rs ) {
+ // List<Object> r = getSelectionResults( rs );
+ // if ( ( r != null ) && ( r.size() > 0 ) ) {
+ // return r.get( 0 );
+ // }
+ // return null;
+ // }
+
+
+ private static String decode( String input ) {
+ try {
+ return URLDecoder.decode( input, "UTF-8" );
+ }
+ catch ( UnsupportedEncodingException e ) {
+ // shouldn't happen, but just in case
+ throw new RuntimeException( e );
+ }
+ }
+
+
+ // note: very likely to be null
+ public String getCollection() {
+ return collection;
+ }
+
+
+ public void setCollection( String collection ) {
+ this.collection = collection;
+ }
+
+
+ // may be null
+ public Optional<String> getQl() {
+
+ //if a query exists, but with no ql, we select all
+ return Optional.fromNullable( ql );
+ }
+
+
++ /**
++ * Return true if no query is present and we should perform a graph search
++ * @return
++ */
++ public boolean graphSearch(){
++ return ql == null;
++ }
++
+ public Query setQl( String ql ) {
+ this.ql = ql;
+ return this;
+ }
+
+
+ /**
+ * Get the connection type
+ * @return
+ */
+ public String getConnectionType() {
+ return connectionType;
+ }
+
+
+ /**
+ * Set the connection type
+ * @param connection
+ * @return
+ */
+ public Query setConnectionType( final String connection ) {
+ this.connectionType = connection;
+ return this;
+ }
+
+
+ public String getType() {
+ return type;
+ }
+
+
+ public Level getLevel() {
+ return level;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index 03f1df8,13187a7..10ee0f6
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@@ -28,11 -28,8 +28,9 @@@ import java.util.UUID
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
- import org.apache.usergrid.persistence.core.guice.ProxyImpl;
- import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index fcab21f,eb59625..7f2ff09
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@@ -89,26 -76,19 +76,22 @@@ public interface IndexFig extends Guicy
/**
- * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ * Timeout calls to elasticsearch.
+ * @return
*/
- public static final String ELASTICSEARCH_QUEUE_OFFER_TIMEOUT = "elasticsearch.queue.offer_timeout";
-
+ String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
- public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+ String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
/**
- * Timeout calls to elasticsearch.
- * @return
+ * The client type to use. Valid values are NODE or TRANSPORT
*/
- public static final String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
+ String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
- public static final String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
+
++ String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+
+
@Default( "127.0.0.1" )
@Key( ELASTICSEARCH_HOSTS )
String getHosts();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/pom.xml
----------------------------------------------------------------------
[18/20] incubator-usergrid git commit: Query by edges collection
working.
Posted by to...@apache.org.
Query by edges collection working.
ES Query by collection not working
Cursor generation not implemented
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/23acdd50
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/23acdd50
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/23acdd50
Branch: refs/heads/two-dot-o-dev
Commit: 23acdd509ffd5b423d7cdf5c58a16ca062ab9e5a
Parents: 9dc65dc
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 27 10:38:06 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 27 10:38:06 2015 -0600
----------------------------------------------------------------------
.../usergrid/corepersistence/CoreModule.java | 4 +
.../corepersistence/CpEntityManager.java | 15 +-
.../corepersistence/CpEntityManagerFactory.java | 11 +-
.../corepersistence/CpRelationManager.java | 11 +-
.../pipeline/PipelineModule.java | 17 ++-
.../corepersistence/pipeline/read/Filter.java | 6 +-
.../pipeline/read/FilterFactory.java | 102 -------------
.../pipeline/read/ReadFilterFactory.java | 102 +++++++++++++
.../pipeline/read/ReadFilterFactoryImpl.java | 152 +++++++++++++++++++
.../pipeline/read/ReadPipelineBuilder.java | 3 +-
.../pipeline/read/ReadPipelineBuilderImpl.java | 31 ++--
...stractQueryElasticSearchCollectorFilter.java | 14 +-
...yCollectionElasticSearchCollectorFilter.java | 20 ++-
...yConnectionElasticSearchCollectorFilter.java | 27 ++--
.../read/entity/EntityLoadCollectorFilter.java | 11 +-
.../read/graph/AbstractReadGraphFilter.java | 36 ++---
.../graph/ReadGraphCollectionByIdFilter.java | 2 +
.../read/graph/ReadGraphCollectionFilter.java | 4 +-
.../graph/ReadGraphConnectionByIdFilter.java | 2 +
.../graph/ReadGraphConnectionByTypeFilter.java | 8 +-
.../read/graph/ReadGraphConnectionFilter.java | 4 +-
.../results/ObservableQueryExecutor.java | 3 +-
.../corepersistence/util/CpNamingUtils.java | 20 +--
.../graph/test/util/EdgeTestUtils.java | 2 +-
.../setup/ConcurrentProcessSingleton.java | 6 +-
25 files changed, 390 insertions(+), 223 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
index 06fe058..ca20e23 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CoreModule.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.corepersistence.migration.CoreMigration;
import org.apache.usergrid.corepersistence.migration.CoreMigrationPlugin;
import org.apache.usergrid.corepersistence.migration.EntityTypeMappingMigration;
import org.apache.usergrid.corepersistence.migration.MigrationModuleVersionPlugin;
+import org.apache.usergrid.corepersistence.pipeline.PipelineModule;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservable;
import org.apache.usergrid.corepersistence.rx.impl.AllApplicationsObservableImpl;
import org.apache.usergrid.corepersistence.rx.impl.AllEntitiesInSystemImpl;
@@ -158,6 +159,9 @@ public class CoreModule extends AbstractModule {
install( new GuicyFigModule( ApplicationIdCacheFig.class ) );
+ //install our pipeline modules
+ install(new PipelineModule());
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 748a069..2d419b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -38,6 +38,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.index.AsyncIndexService;
+import org.apache.usergrid.corepersistence.pipeline.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
@@ -179,9 +180,7 @@ public class CpEntityManager implements EntityManager {
private final AsyncIndexService indexService;
-
- private EntityCollectionManagerFactory entityCollectionManagerFactory;
- private GraphManagerFactory graphManagerFactory;
+ private PipelineBuilderFactory pipelineBuilderFactory;
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
@@ -224,16 +223,16 @@ public class CpEntityManager implements EntityManager {
* @param graphManagerFactory
*/
public CpEntityManager( final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache,
- final MetricsFactory metricsFactory,final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final GraphManagerFactory graphManagerFactory , final UUID applicationId ) {
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.graphManagerFactory = graphManagerFactory;
+ final MetricsFactory metricsFactory,final PipelineBuilderFactory pipelineBuilderFactory , final UUID applicationId ) {
+
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
Preconditions.checkNotNull( indexService, "indexService must not be null" );
+ Preconditions.checkNotNull( pipelineBuilderFactory, "pipelineBuilderFactory must not be null" );
+ this.pipelineBuilderFactory = pipelineBuilderFactory;
this.managerCache = managerCache;
@@ -747,7 +746,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager relationManager =
- new CpRelationManager( metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory, indexService, this, applicationId, entityRef );
+ new CpRelationManager( metricsFactory, managerCache, pipelineBuilderFactory, indexService, this, applicationId, entityRef );
return relationManager;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f06cf40..0d10fd2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -35,6 +35,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.index.ReIndexService;
+import org.apache.usergrid.corepersistence.pipeline.PipelineBuilderFactory;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
import org.apache.usergrid.persistence.AbstractEntity;
@@ -74,6 +75,7 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
+import com.amazonaws.services.elastictranscoder.model.Pipeline;
import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
@@ -118,8 +120,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final ApplicationIdCache applicationIdCache;
private ManagerCache managerCache;
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private final GraphManagerFactory graphManagerFactory;
private CassandraService cassandraService;
private CounterUtils counterUtils;
@@ -127,6 +127,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final EntityIndex entityIndex;
private final MetricsFactory metricsFactory;
private final AsyncIndexService indexService;
+ private final PipelineBuilderFactory pipelineBuilderFactory;
public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
final Injector injector ) {
@@ -139,10 +140,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
this.indexService = injector.getInstance( AsyncIndexService.class );
+ this.pipelineBuilderFactory = injector.getInstance( PipelineBuilderFactory.class );
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
- this.entityCollectionManagerFactory = injector.getInstance( EntityCollectionManagerFactory.class );
- this.graphManagerFactory = injector.getInstance( GraphManagerFactory.class );
}
@@ -199,8 +199,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory,
- entityCollectionManagerFactory, graphManagerFactory, applicationId );
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, pipelineBuilderFactory, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index af926bf..e7ef0ff 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -109,9 +109,7 @@ public class CpRelationManager implements RelationManager {
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
private ManagerCache managerCache;
- private EntityCollectionManagerFactory entityCollectionManagerFactory;
- private GraphManagerFactory graphManagerFactory;
- private PipelineBuilderFactory pipelineBuilderFactory;
+ private final PipelineBuilderFactory pipelineBuilderFactory;
private EntityManager em;
@@ -130,8 +128,7 @@ public class CpRelationManager implements RelationManager {
public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache,
- final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final GraphManagerFactory graphManagerFactory, final AsyncIndexService indexService,
+ final PipelineBuilderFactory pipelineBuilderFactory, final AsyncIndexService indexService,
final EntityManager em, final UUID applicationId, final EntityRef headEntity ) {
@@ -148,9 +145,7 @@ public class CpRelationManager implements RelationManager {
this.headEntity = headEntity;
this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
-
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.graphManagerFactory = graphManagerFactory;
+ this.pipelineBuilderFactory = pipelineBuilderFactory;
this.metricsFactory = metricsFactory;
this.updateCollectionTimer =
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
index 8afe2f8..55b84af 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
@@ -20,7 +20,10 @@
package org.apache.usergrid.corepersistence.pipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.ReadFilterFactory;
+import org.apache.usergrid.corepersistence.pipeline.read.ReadFilterFactoryImpl;
import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
+import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilderImpl;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
@@ -35,6 +38,18 @@ public class PipelineModule extends AbstractModule {
protected void configure() {
//Use Guice to create the builder since we don't really need to do anything
//other than DI when creating the filters
- install(new FactoryModuleBuilder().build( ReadPipelineBuilder.class ));
+ bind( ReadFilterFactory.class ).to( ReadFilterFactoryImpl.class );
+
+
+ //Use Guice to create the builder since we don't really need to do anything
+ //other than DI when creating the filters
+ install( new FactoryModuleBuilder().implement( ReadPipelineBuilder.class, ReadPipelineBuilderImpl.class )
+ .build( PipelineBuilderFactory.class ) );
+
+
+
+ //Use Guice to create the builder since we don't really need to do anything
+ //other than DI when creating the filters
+// install( new FactoryModuleBuilder().build( ReadFilterFactory.class ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
index 670d2ed..f50a2f4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
@@ -37,12 +37,12 @@ public interface Filter<T> extends Observable.Transformer<Id, T> {
/**
- * Set the id of this command in it's execution environment
+ * Set the id of this filter in it's execution environment
*/
void setId( final int id );
/**
- * Set the cursor cache into the command
+ * Set the cursor cache into the filter
*
* @param readCache Set the cache that was used in the request
* @param writeCache Set the cache to be used when writing the results
@@ -50,7 +50,7 @@ public interface Filter<T> extends Observable.Transformer<Id, T> {
void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache );
/**
- * Set the application scope of the command
+ * Set the application scope of the filter
* @param applicationScope
*/
void setApplicationScope(final ApplicationScope applicationScope);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
deleted file mode 100644
index 8f9776e..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.pipeline.read;
-
-
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryCollectionElasticSearchCollectorFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryConnectionElasticSearchCollectorFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionByIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByIdFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByTypeFilter;
-import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-
-/**
- * A factory for generating read commands
- */
-public interface FilterFactory {
-
-
- /**
- * Generate a new instance of the command with the specified parameters
- */
- ReadGraphCollectionFilter readGraphCollectionCommand( final String collectionName );
-
- /**
- * Read a connection between two entities, the incoming and the target entity
- * @param collectionName
- * @param targetId
- * @return
- */
- ReadGraphCollectionByIdFilter readGraphCollectionByIdFilter(final String collectionName, final Id targetId);
-
- /**
- * Generate a new instance of the command with the specified parameters
- */
- ReadGraphConnectionFilter readGraphConnectionCommand( final String connectionName );
-
- /**
- * Generate a new instance of the command with the specified parameters
- */
- ReadGraphConnectionByTypeFilter readGraphConnectionCommand( final String connectionName, final String entityType );
-
-
- /**
- * Read a connection directly between two identifiers
- * @param connectionName
- * @param targetId
- * @return
- */
- ReadGraphConnectionByIdFilter readGraphConnectionByIdFilter(final String connectionName, final Id targetId);
-
- /**
- * Generate a new instance of the command with the specified parameters
- */
- EntityLoadCollectorFilter entityLoadCollector();
-
- /**
- * Generate a new instance of the command with the specified parameters
- */
- QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector( final String collectionName, final String query);
-
-
- /**
- * Generate a new instance of the command with the specified parameters
- */
- QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName,final String query);
-
-
- /**
- * Generate a new instance of the command with the specified parameters
- */
- QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName, final String connectionEntityType, final String query );
-
-
- /**
- * Get an entity id filter. Used as a 1.0->2.0 bridge since we're not doing full traversals
- */
- EntityIdFilter getEntityIdFilter( final Id entityId );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactory.java
new file mode 100644
index 0000000..92bdacb
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactory.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryCollectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryConnectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByTypeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A factory for generating read commands
+ */
+public interface ReadFilterFactory {
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ ReadGraphCollectionFilter readGraphCollectionCommand( final String collectionName );
+
+ /**
+ * Read a connection between two entities, the incoming and the target entity
+ * @param collectionName
+ * @param targetId
+ * @return
+ */
+ ReadGraphCollectionByIdFilter readGraphCollectionByIdFilter(final String collectionName, final Id targetId);
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ ReadGraphConnectionFilter readGraphConnectionCommand( final String connectionName );
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ ReadGraphConnectionByTypeFilter readGraphConnectionCommand( final String connectionName, final String entityType );
+
+
+ /**
+ * Read a connection directly between two identifiers
+ * @param connectionName
+ * @param targetId
+ * @return
+ */
+ ReadGraphConnectionByIdFilter readGraphConnectionByIdFilter(final String connectionName, final Id targetId);
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ EntityLoadCollectorFilter entityLoadCollector();
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector( final String collectionName, final String query);
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName,final String query);
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName, final String connectionEntityType, final String query );
+
+
+ /**
+ * Get an entity id filter. Used as a 1.0->2.0 bridge since we're not doing full traversals
+ */
+ EntityIdFilter getEntityIdFilter( final Id entityId );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java
new file mode 100644
index 0000000..19162bb
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadFilterFactoryImpl.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryCollectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryConnectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByTypeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class ReadFilterFactoryImpl implements ReadFilterFactory {
+
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final EntityIndexFactory entityIndexFactory;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public ReadFilterFactoryImpl( final GraphManagerFactory graphManagerFactory,
+ final EntityIndexFactory entityIndexFactory,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+
+
+ this.graphManagerFactory = graphManagerFactory;
+ this.entityIndexFactory = entityIndexFactory;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public ReadGraphCollectionFilter readGraphCollectionCommand( final String collectionName ) {
+ return new ReadGraphCollectionFilter( graphManagerFactory, collectionName );
+ }
+
+
+ @Override
+ public ReadGraphCollectionByIdFilter readGraphCollectionByIdFilter( final String collectionName,
+ final Id targetId ) {
+ return new ReadGraphCollectionByIdFilter( graphManagerFactory, collectionName, targetId );
+ }
+
+
+ @Override
+ public ReadGraphConnectionFilter readGraphConnectionCommand( final String connectionName ) {
+ return new ReadGraphConnectionFilter( graphManagerFactory, connectionName );
+ }
+
+
+ @Override
+ public ReadGraphConnectionByTypeFilter readGraphConnectionCommand( final String connectionName,
+ final String entityType ) {
+ return new ReadGraphConnectionByTypeFilter( graphManagerFactory, connectionName, entityType );
+ }
+
+
+ @Override
+ public ReadGraphConnectionByIdFilter readGraphConnectionByIdFilter( final String connectionName,
+ final Id targetId ) {
+ return new ReadGraphConnectionByIdFilter( graphManagerFactory, connectionName, targetId );
+ }
+
+
+ @Override
+ public EntityLoadCollectorFilter entityLoadCollector() {
+ return new EntityLoadCollectorFilter( entityCollectionManagerFactory );
+ }
+
+
+ /**
+ * TODO refactor these impls to use RX internally, as well as remove the query object
+ */
+ @Override
+ public QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector(
+ final String collectionName, final String query ) {
+
+ final Query queryObject = Query.fromQL( query );
+
+ final QueryCollectionElasticSearchCollectorFilter filter =
+ new QueryCollectionElasticSearchCollectorFilter( entityCollectionManagerFactory, entityIndexFactory,
+ collectionName, queryObject );
+
+ return filter;
+ }
+
+
+ @Override
+ public QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector(
+ final String connectionName, final String query ) {
+
+ final Query queryObject = Query.fromQL( query );
+
+ final QueryConnectionElasticSearchCollectorFilter filter =
+ new QueryConnectionElasticSearchCollectorFilter( entityCollectionManagerFactory, entityIndexFactory,
+ connectionName, queryObject );
+
+ return filter;
+ }
+
+
+ @Override
+ public QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector(
+ final String connectionName, final String connectionEntityType, final String query ) {
+
+ final Query queryObject = Query.fromQL( query );
+ queryObject.setConnectionType( connectionEntityType );
+
+ final QueryConnectionElasticSearchCollectorFilter filter =
+ new QueryConnectionElasticSearchCollectorFilter( entityCollectionManagerFactory, entityIndexFactory,
+ connectionName, queryObject );
+
+ return filter;
+ }
+
+
+ @Override
+ public EntityIdFilter getEntityIdFilter( final Id entityId ) {
+ return new EntityIdFilter( entityId );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
index d40cb12..5d83dac 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
@@ -54,6 +54,7 @@ public interface ReadPipelineBuilder {
*/
ReadPipelineBuilder setStartId(final Id id);
+
/**
* Add a get entity to the pipeline
*/
@@ -92,7 +93,7 @@ public interface ReadPipelineBuilder {
/**
- * Get all entities in a connection with a query
+ * Get all entities in a connection with a query and a target entity type
*/
ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType, final String query);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
index 7319f21..7ffe957 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
@@ -39,7 +39,7 @@ import rx.Observable;
public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
- private final FilterFactory filterFactory;
+ private final ReadFilterFactory readFilterFactory;
private final DataPipeline pipeline;
@@ -54,11 +54,16 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Inject
- public ReadPipelineBuilderImpl( final FilterFactory filterFactory,
+ public ReadPipelineBuilderImpl( final ReadFilterFactory readFilterFactory,
@Assisted final ApplicationScope applicationScope ) {
- this.filterFactory = filterFactory;
+ this.readFilterFactory = readFilterFactory;
+
+ //set up our pipeline with our application scope
this.pipeline = new DataPipeline( applicationScope );
+
+ //init our cursor to empty
this.cursor = Optional.absent();
+
//set the default limit
this.limit = Optional.absent();
}
@@ -84,7 +89,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
public ReadPipelineBuilder setStartId( final Id id ) {
- pipeline.withTraverseCommand( filterFactory.getEntityIdFilter( id ) );
+ pipeline.withTraverseCommand( readFilterFactory.getEntityIdFilter( id ) );
this.collectorFilter = null;
@@ -96,7 +101,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
public ReadPipelineBuilder getEntityViaCollection( final String collectionName, final Id entityId ) {
- pipeline.withTraverseCommand( filterFactory.readGraphCollectionByIdFilter( collectionName, entityId ) );
+ pipeline.withTraverseCommand( readFilterFactory.readGraphCollectionByIdFilter( collectionName, entityId ) );
setEntityLoaderFilter();
@@ -108,7 +113,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
public ReadPipelineBuilder getCollection( final String collectionName ) {
- pipeline.withTraverseCommand( filterFactory.readGraphCollectionCommand( collectionName ) );
+ pipeline.withTraverseCommand( readFilterFactory.readGraphCollectionCommand( collectionName ) );
setEntityLoaderFilter();
@@ -120,7 +125,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
public ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query ) {
//TODO, this should really be 2 a TraverseFilter with an entityLoad collector
- collectorFilter = filterFactory.queryCollectionElasticSearchCollector( collectionName, query );
+ collectorFilter = readFilterFactory.queryCollectionElasticSearchCollector( collectionName, query );
return this;
}
@@ -128,7 +133,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
public ReadPipelineBuilder getEntityViaConnection( final String connectionName, final Id entityId ) {
- pipeline.withTraverseCommand( filterFactory.readGraphConnectionByIdFilter( connectionName, entityId ) );
+ pipeline.withTraverseCommand( readFilterFactory.readGraphConnectionByIdFilter( connectionName, entityId ) );
setEntityLoaderFilter();
return this;
@@ -138,7 +143,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
public ReadPipelineBuilder getConnection( final String connectionName ) {
- pipeline.withTraverseCommand( filterFactory.readGraphConnectionCommand( connectionName ) );
+ pipeline.withTraverseCommand( readFilterFactory.readGraphConnectionCommand( connectionName ) );
setEntityLoaderFilter();
return this;
@@ -147,7 +152,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
@Override
public ReadPipelineBuilder getConnection( final String connectionName, final String entityType ) {
- pipeline.withTraverseCommand( filterFactory.readGraphConnectionCommand( connectionName, entityType ) );
+ pipeline.withTraverseCommand( readFilterFactory.readGraphConnectionCommand( connectionName, entityType ) );
setEntityLoaderFilter();
return this;
@@ -164,7 +169,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query ) {
//TODO, this should really be 2 a TraverseFilter with an entityLoad collector
- collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, query );
+ collectorFilter = readFilterFactory.queryConnectionElasticSearchCollector( connectionName, query );
return this;
}
@@ -176,7 +181,7 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
//TODO, this should really be 2 a TraverseFilter with an entityLoad collector
collectorFilter =
- filterFactory.queryConnectionElasticSearchCollector( connectionName, entityType, query);
+ readFilterFactory.queryConnectionElasticSearchCollector( connectionName, entityType, query);
return this;
}
@@ -192,6 +197,6 @@ public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
private void setEntityLoaderFilter() {
- collectorFilter = filterFactory.entityLoadCollector();
+ collectorFilter = readFilterFactory.entityLoadCollector();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
index 01ab21f..a843f51 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
@@ -29,8 +29,8 @@ import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -54,14 +54,14 @@ public abstract class AbstractQueryElasticSearchCollectorFilter extends Abstract
implements CollectorFilter<Results> {
- protected final ApplicationEntityIndex entityIndex;
+ protected final EntityIndexFactory applicationEntityIndex;
protected final Query query;
private int limit;
@Inject
- protected AbstractQueryElasticSearchCollectorFilter( final ApplicationEntityIndex entityIndex, final Query query ) {
- this.entityIndex = entityIndex;
+ protected AbstractQueryElasticSearchCollectorFilter( final EntityIndexFactory applicationEntityIndex, final Query query ) {
+ this.applicationEntityIndex = applicationEntityIndex;
this.query = query;
}
@@ -69,6 +69,10 @@ public abstract class AbstractQueryElasticSearchCollectorFilter extends Abstract
@Override
public Observable<Results> call( final Observable<Id> idObservable ) {
+
+ final ApplicationEntityIndex
+ entityIndex = applicationEntityIndex.createApplicationEntityIndex( applicationScope );
+
return idObservable.flatMap( id -> {
//TODO, refactor this logic to use Observables. make this a TraverseFilter and load entities with the entity loader collector
@@ -76,6 +80,8 @@ public abstract class AbstractQueryElasticSearchCollectorFilter extends Abstract
final SearchEdge searchEdge = getSearchEdge( id );
final SearchTypes searchTypes = getSearchTypes();
+
+
final Iterable<Results> executor =
new ElasticSearchQueryExecutor( resultsLoaderFactory, entityIndex, applicationScope,
searchEdge, searchTypes, query.withLimit( limit ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
index f0e8139..4813978 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
@@ -25,7 +25,9 @@ import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.Resu
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -41,18 +43,18 @@ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createColle
*/
public class QueryCollectionElasticSearchCollectorFilter extends AbstractQueryElasticSearchCollectorFilter {
- private final EntityCollectionManager entityCollectionManager;
- private final ApplicationEntityIndex applicationEntityIndex;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final EntityIndexFactory entityIndexFactory;
private final String collectionName;
@Inject
- protected QueryCollectionElasticSearchCollectorFilter( final EntityCollectionManager entityCollectionManager,
- final ApplicationEntityIndex applicationEntityIndex,
+ public QueryCollectionElasticSearchCollectorFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final EntityIndexFactory entityIndexFactory,
@Assisted final String collectionName , @Assisted final Query query ) {
- super( applicationEntityIndex, query );
- this.entityCollectionManager = entityCollectionManager;
- this.applicationEntityIndex = applicationEntityIndex;
+ super( entityIndexFactory, query );
+ this.entityIndexFactory = entityIndexFactory;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
this.collectionName = collectionName;
}
@@ -75,9 +77,11 @@ public class QueryCollectionElasticSearchCollectorFilter extends AbstractQueryEl
@Override
protected ResultsLoaderFactory getResultsLoaderFactory( final Id id ) {
+ final EntityCollectionManager entityCollectionManager = entityCollectionManagerFactory.createCollectionManager( applicationScope );
+ final ApplicationEntityIndex entityIndex = entityIndexFactory.createApplicationEntityIndex( applicationScope );
final EntityRef entityRef = getRef( id );
- return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, applicationEntityIndex, entityRef,
+ return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, entityIndex, entityRef,
collectionName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
index 2b380df..2f7a6b3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
@@ -25,7 +25,9 @@ import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.Resu
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -41,19 +43,19 @@ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConne
*/
public class QueryConnectionElasticSearchCollectorFilter extends AbstractQueryElasticSearchCollectorFilter {
- private final EntityCollectionManager entityCollectionManager;
- private final ApplicationEntityIndex applicationEntityIndex;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final EntityIndexFactory entityIndexFactory;
private final String connectionName;
@Inject
- protected QueryConnectionElasticSearchCollectorFilter( final EntityCollectionManager entityCollectionManager,
- final ApplicationEntityIndex applicationEntityIndex,
- @Assisted final String connectionName,
- @Assisted final Query query ) {
- super( applicationEntityIndex, query );
- this.entityCollectionManager = entityCollectionManager;
- this.applicationEntityIndex = applicationEntityIndex;
+ public QueryConnectionElasticSearchCollectorFilter(
+ final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final EntityIndexFactory entityIndexFactory, @Assisted final String connectionName,
+ @Assisted final Query query ) {
+ super( entityIndexFactory, query );
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.entityIndexFactory = entityIndexFactory;
this.connectionName = connectionName;
}
@@ -77,8 +79,13 @@ public class QueryConnectionElasticSearchCollectorFilter extends AbstractQueryEl
@Override
protected ResultsLoaderFactory getResultsLoaderFactory( final Id id ) {
+
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+ final ApplicationEntityIndex entityIndex = entityIndexFactory.createApplicationEntityIndex( applicationScope );
+
final EntityRef entityRef = getRef( id );
- return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, applicationEntityIndex, entityRef,
+ return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, entityIndex, entityRef,
connectionName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java
index 74f626d..3b140ae 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java
@@ -50,15 +50,12 @@ public class EntityLoadCollectorFilter extends AbstractFilter<Results, Serializa
implements CollectorFilter<Results> {
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
- private final ApplicationScope applicationScope;
private int resultSize;
@Inject
- public EntityLoadCollectorFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final ApplicationScope applicationScope ) {
+ public EntityLoadCollectorFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.applicationScope = applicationScope;
}
@@ -92,7 +89,7 @@ public class EntityLoadCollectorFilter extends AbstractFilter<Results, Serializa
final Observable<MvccEntity> mvccEntityObservable = Observable.from( entitySet.getEntities() );
//convert them to our old entity model, then filter nulls, meaning they weren't found
- return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter( entity -> entity == null )
+ return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter( entity -> entity != null )
//convert them to a list, then map them into results
.toList().map( entities -> {
@@ -100,9 +97,7 @@ public class EntityLoadCollectorFilter extends AbstractFilter<Results, Serializa
results.setCursor( generateCursor() );
return results;
- } )
- //if no results are present, return an empty results
- .singleOrDefault( new Results( ) );
+ } );
} );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
index 4bdcdad..06aae83 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
@@ -32,22 +32,18 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
/**
* Command for reading graph edges
*/
-public abstract class AbstractReadGraphFilter extends AbstractFilter<Id, Edge>
- implements TraverseFilter {
+public abstract class AbstractReadGraphFilter extends AbstractFilter<Id, Edge> implements TraverseFilter {
private final GraphManagerFactory graphManagerFactory;
/**
* Create a new instance of our command
- * @param graphManagerFactory
*/
public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory ) {
this.graphManagerFactory = graphManagerFactory;
@@ -67,20 +63,20 @@ public abstract class AbstractReadGraphFilter extends AbstractFilter<Id, Edge>
//return all ids that are emitted from this edge
- return observable.flatMap( new Func1<Id, Observable<Id>>() {
-
- @Override
- public Observable<Id> call( final Id id ) {
-
- final SimpleSearchByEdgeType search = new SimpleSearchByEdgeType(id,edgeName, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, startFromCursor );
-
- /**
- * TODO, pass a message with pointers to our cursor values to be generated later
- */
- return graphManager.loadEdgesFromSource( search ).doOnNext( edge -> setCursor( edge ) ).map(
- edge -> edge.getTargetNode() );
- }
+ return observable.flatMap( id -> {
+
+ final SimpleSearchByEdgeType search =
+ new SimpleSearchByEdgeType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ startFromCursor );
+
+ /**
+ * TODO, pass a message with pointers to our cursor values to be generated later
+ */
+ return graphManager.loadEdgesFromSource( search )
+ //set our cursor every edge we traverse
+ .doOnNext( edge -> setCursor( edge ) )
+ //map our id from the target edge
+ .map( edge -> edge.getTargetNode() );
} );
}
@@ -91,10 +87,8 @@ public abstract class AbstractReadGraphFilter extends AbstractFilter<Id, Edge>
}
-
/**
* Get the edge type name we should use when traversing
- * @return
*/
protected abstract String getEdgeTypeName();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
index 2ccad67..da6ad29 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -34,6 +35,7 @@ public class ReadGraphCollectionByIdFilter extends AbstractReadGraphEdgeByIdFilt
private final String collectionName;
+ @Inject
public ReadGraphCollectionByIdFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName, @Assisted final Id targetId ) {
super( graphManagerFactory, targetId );
this.collectionName = collectionName;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
index 3f0a70a..91ae7c3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
@@ -25,7 +25,7 @@ import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromCollectionName;
/**
@@ -48,6 +48,6 @@ public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
@Override
protected String getEdgeTypeName() {
- return getCollectionScopeNameFromCollectionName( collectionName );
+ return getEdgeTypeFromCollectionName( collectionName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
index f16867d..4756d33 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
@@ -34,6 +35,7 @@ public class ReadGraphConnectionByIdFilter extends AbstractReadGraphEdgeByIdFilt
private final String connectionName;
+ @Inject
public ReadGraphConnectionByIdFilter( final GraphManagerFactory graphManagerFactory,
@Assisted final String connectionName, @Assisted final Id targetId ) {
super( graphManagerFactory, targetId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
index 3057111..d5cdd66 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
@@ -30,13 +30,12 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
+import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromConnectionType;
/**
@@ -52,6 +51,7 @@ public class ReadGraphConnectionByTypeFilter extends AbstractFilter<Id, Edge> im
/**
* Create a new instance of our command
*/
+ @Inject
public ReadGraphConnectionByTypeFilter( final GraphManagerFactory graphManagerFactory,
@Assisted final String connectionName, @Assisted final String entityType ) {
this.graphManagerFactory = graphManagerFactory;
@@ -69,7 +69,7 @@ public class ReadGraphConnectionByTypeFilter extends AbstractFilter<Id, Edge> im
//set our our constant state
final Optional<Edge> startFromCursor = getCursor();
- final String edgeName = getConnectionScopeName( connectionName );
+ final String edgeName = getEdgeTypeFromConnectionType( connectionName );
//return all ids that are emitted from this edge
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
index 49360f6..0d4971b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
@@ -25,7 +25,7 @@ import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getEdgeTypeFromConnectionType;
/**
@@ -48,6 +48,6 @@ public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
@Override
protected String getEdgeTypeName() {
- return getConnectionScopeName( connectionName );
+ return getEdgeTypeFromConnectionType( connectionName );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
index 382fb64..f5a2893 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
@@ -40,7 +40,8 @@ public class ObservableQueryExecutor implements QueryExecutor {
public ObservableQueryExecutor( final Observable<Results> resultsObservable ) {
- this.resultsObservable = resultsObservable;
+ //in no values, we must emit an empty results so add the default
+ this.resultsObservable = resultsObservable.defaultIfEmpty( new Results() );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index 5472071..c65db09 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -236,10 +236,10 @@ public class CpNamingUtils {
if ( isCollectionEdgeType( edgeName ) ) {
- return getCollectionScopeNameFromCollectionName( getCollectionName( edgeName ) );
+ return getCollectionName( edgeName ) ;
}
- return getConnectionScopeName( getConnectionType( edgeName ) );
+ return getConnectionType( edgeName ) ;
}
@@ -260,20 +260,4 @@ public class CpNamingUtils {
}
- /**
- * Only use when searching graph. When searching ES use the <see>createCollectionSearchEdge</see>
- */
- public static String getCollectionScopeNameFromCollectionName( String name ) {
- String csn = EDGE_COLL_SUFFIX + name;
- return csn.toLowerCase();
- }
-
-
- /**
- * Only use when searching graph. When searching ES use the <see>createConnectionSearchByEdge</see>
- */
- public static String getConnectionScopeName( String connectionType ) {
- String csn = EDGE_CONN_SUFFIX + connectionType;
- return csn.toLowerCase();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
index c81fa58..0239f4b 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
@@ -164,7 +164,7 @@ public class EdgeTestUtils {
*/
public static SearchByIdType createSearchByEdgeAndId( final Id sourceId, final String type, final long maxVersion,
final String idType, final Edge last ) {
- return new SimpleSearchByIdType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, idType, last );
+ return new SimpleSearchByIdType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, idType, Optional.of(last) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/23acdd50/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java b/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
index 7cb5073..214334b 100644
--- a/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
@@ -43,8 +43,10 @@ public class ConcurrentProcessSingleton {
public static final int LOCK_PORT = Integer.parseInt(
System.getProperty( "test.lock.port", "10101" ) );
- public static final boolean CLEAN_STORAGE =
- Boolean.parseBoolean( System.getProperty( "test.clean.storage", "false" ) );
+// public static final boolean CLEAN_STORAGE =
+// Boolean.parseBoolean( System.getProperty( "test.clean.storage", "false" ) );
+
+ public static final boolean CLEAN_STORAGE = false;
public static final long ONE_MINUTE = 60000;
[05/20] incubator-usergrid git commit: Initial setup of command
pattern and wiring into legacy code
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index ce45ebf..b34bd93 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -41,6 +41,7 @@ import org.slf4j.LoggerFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.util.Assert;
+import org.apache.usergrid.corepersistence.ManagerCache;
import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
@@ -69,6 +70,8 @@ import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.CounterUtils.AggregateCounterSelection;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.entities.Event;
import org.apache.usergrid.persistence.entities.Group;
@@ -78,6 +81,7 @@ import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsE
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
@@ -193,6 +197,7 @@ public class EntityManagerImpl implements EntityManager {
@Resource
private EntityManagerFactoryImpl emf;
+
@Resource
private QueueManagerFactoryImpl qmf;
@Resource
@@ -212,10 +217,6 @@ public class EntityManagerImpl implements EntityManager {
}
- @Override
- public void init(EntityManagerFactory emf, UUID applicationId) {
- init( (EntityManagerFactoryImpl)emf, null, null, applicationId, false);
- }
public EntityManager init(
@@ -523,7 +524,7 @@ public class EntityManagerImpl implements EntityManager {
*/
Set<UUID> ownerEntityIds = getUUIDsForUniqueProperty(
- new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), entityType, propertyName, propertyValue );
+ new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), entityType, propertyName, propertyValue );
//if there are no entities for this property, we know it's unique. If there are,
// we have to make sure the one we were passed is in the set. otherwise it belongs
@@ -651,7 +652,7 @@ public class EntityManagerImpl implements EntityManager {
@Override
public Map<String, EntityRef> getAlias( String aliasType, List<String> aliases ) throws Exception {
- return getAlias( new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, aliases );
+ return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, aliases );
}
@@ -1249,9 +1250,9 @@ public class EntityManagerImpl implements EntityManager {
HColumn<ByteBuffer, ByteBuffer> result =
cass.getColumn( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- key( entity.getUuid(), dictionaryName ),
- entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ),
- be, be );
+ key( entity.getUuid(), dictionaryName ),
+ entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ), be,
+ be );
if ( result != null ) {
if ( entityHasDictionary && coTypeIsBasic ) {
value = object( dictionaryCoType, result.getValue() );
@@ -1295,7 +1296,7 @@ public class EntityManagerImpl implements EntityManager {
ColumnSlice<ByteBuffer, ByteBuffer> results =
cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- key( entity.getUuid(), dictionaryName ), columnNames, be, be );
+ key( entity.getUuid(), dictionaryName ), columnNames, be, be );
if ( results != null ) {
values = new HashMap<String, Object>();
for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) {
@@ -1352,7 +1353,7 @@ public class EntityManagerImpl implements EntityManager {
List<HColumn<ByteBuffer, ByteBuffer>> results =
cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- key( entity.getUuid(), dictionaryName ), be, be );
+ key( entity.getUuid(), dictionaryName ), be, be );
for ( HColumn<ByteBuffer, ByteBuffer> result : results ) {
Object name = null;
if ( entityHasDictionary ) {
@@ -2296,7 +2297,7 @@ public class EntityManagerImpl implements EntityManager {
Mutator<ByteBuffer> batch = CountingMutator.createFlushingMutator( cass.getApplicationKeyspace( applicationId ),
be );
addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES, getRolePermissionsKey( roleName ), permission,
- ByteBuffer.allocate( 0 ), timestamp );
+ ByteBuffer.allocate( 0 ), timestamp );
batchExecute( batch, CassandraService.RETRY_COUNT );
}
@@ -2377,7 +2378,7 @@ public class EntityManagerImpl implements EntityManager {
Mutator<ByteBuffer> batch = CountingMutator.createFlushingMutator( cass.getApplicationKeyspace( applicationId ),
be );
addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES, getRolePermissionsKey( groupId, roleName ),
- permission, ByteBuffer.allocate( 0 ), timestamp );
+ permission, ByteBuffer.allocate( 0 ), timestamp );
batchExecute( batch, CassandraService.RETRY_COUNT );
}
@@ -2390,7 +2391,7 @@ public class EntityManagerImpl implements EntityManager {
Mutator<ByteBuffer> batch = CountingMutator.createFlushingMutator( cass.getApplicationKeyspace( applicationId ),
be );
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( groupId, roleName ), permission, timestamp );
+ getRolePermissionsKey( groupId, roleName ), permission, timestamp );
batchExecute( batch, CassandraService.RETRY_COUNT );
}
@@ -2422,7 +2423,7 @@ public class EntityManagerImpl implements EntityManager {
@Override
public Map<String, Role> getUserRolesWithTitles( UUID userId ) throws Exception {
return getRolesWithTitles(
- ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) );
+ ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) );
}
@@ -2534,7 +2535,7 @@ public class EntityManagerImpl implements EntityManager {
Mutator<ByteBuffer> m = CountingMutator.createFlushingMutator( cass.getApplicationKeyspace( applicationId ),
be );
counterUtils.batchIncrementAggregateCounters( m, applicationId, userId, groupId, null, category, counters,
- timestamp );
+ timestamp );
batchExecute( m, CassandraService.RETRY_COUNT );
}
@@ -2724,7 +2725,7 @@ public class EntityManagerImpl implements EntityManager {
public Results getConnectingEntities( EntityRef entityRef, String connectionType,
String connectedEntityType, Level resultsLevel ) throws Exception {
- return getConnectingEntities( entityRef, connectionType, connectedEntityType, resultsLevel, 0);
+ return getConnectingEntities( entityRef, connectionType, connectedEntityType, resultsLevel, 0 );
}
@@ -2901,6 +2902,16 @@ public class EntityManagerImpl implements EntityManager {
@Override
+ public void init( final CassandraService cassandraService, final CounterUtils counterUtils,
+ final MetricsFactory metricsFactory, final GraphManagerFactory graphManagerFactory,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final ManagerCache managerCache, final UUID applicationId ) {
+ throw new UnsupportedOperationException( "Removed anyway" );
+
+ }
+
+
+ @Override
public EntityRef getGroupRoleRef( UUID ownerId, String roleName) throws Exception {
return new SimpleEntityRef( Role.ENTITY_TYPE, SimpleRoleRef.getIdForGroupIdAndRoleName( ownerId, roleName ));
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
index 99cfcf5..647424c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
@@ -164,6 +164,9 @@ public class CoreApplication implements Application, TestRule {
LOG.info( "Created new application {} in organization {}", appName, orgName );
+// //wait for the index before proceeding
+// em.refreshIndex();
+
}
protected void before( Description description ) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
index 35b6a12..ffdfc2a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
@@ -18,6 +18,8 @@
*/
package org.apache.usergrid.persistence.collection;
+import java.util.Collection;
+
import org.apache.usergrid.persistence.model.entity.Id;
@@ -31,17 +33,23 @@ public interface EntitySet {
* @param entityId
* @return
*/
- public MvccEntity getEntity(Id entityId);
+ MvccEntity getEntity( Id entityId );
+
+ /**
+ * Get all entities in the set
+ * @return
+ */
+ Collection<MvccEntity> getEntities();
/**
* Get the number of entities in this set
* @return
*/
- public int size();
+ int size();
/**
* Return true if the set is empty
* @return
*/
- public boolean isEmpty();
+ boolean isEmpty();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
index 921093b..126f9f3 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence.collection.serialization.impl;/*
*/
+import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@@ -32,8 +33,7 @@ public class EntitySetImpl implements EntitySet {
private final Map<Id, MvccEntity> entities;
- public EntitySetImpl(
- final int expectedSize ) {
+ public EntitySetImpl(final int expectedSize ) {
this.entities = new HashMap<>( expectedSize );
}
@@ -48,6 +48,12 @@ public class EntitySetImpl implements EntitySet {
return entities.get( entityId );
}
+ public Collection<MvccEntity> getEntities(){
+ return entities.values();
+ }
+
+
+
@Override
public int size() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
index cdad0d1..613585e 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
@@ -94,20 +94,20 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
innerObservers[i] = inner;
}
-
- /**
- * Once we're set up, begin the subscription to sub observables
- */
- for ( int i = 0; i < observables.length; i++ ) {
- //subscribe after setting them up
- //add our subscription to the composite for future cancellation
- Subscription subscription = observables[i].subscribe( innerObservers[i] );
-
- csub.add( subscription );
-
- //add the internal composite subscription
- outerOperation.add( csub );
- }
+// TODO, if this merge makes it into 2.0-dev remove this and use 2.0 -dev
+// /**
+// * Once we're set up, begin the subscription to sub observables
+// */
+// for ( int i = 0; i < observables.length; i++ ) {
+// //subscribe after setting them up
+// //add our subscription to the composite for future cancellation
+// Subscription subscription = observables[i].subscribe( innerObservers[i] );
+//
+// csub.add( subscription );
+//
+// //add the internal composite subscription
+// outerOperation.add( csub );
+// }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/Edge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/Edge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/Edge.java
index 5613ae9..b2d2ced 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/Edge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/Edge.java
@@ -22,8 +22,11 @@ package org.apache.usergrid.persistence.graph;
import java.io.Serializable;
+import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
/**
* Defines a directed edge from the source node to the target node. Edges are considered immutable.
@@ -31,6 +34,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
*
* @author tnine
*/
+@JsonDeserialize(as = SimpleEdge.class)
public interface Edge extends Serializable {
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
index 749130b..5e47ae0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
@@ -70,7 +70,7 @@ public interface SearchByEdgeType {
* Options for ordering. By default, we want to perform descending for common use cases and read speed. This is our our data
* is optimized in cassandra
*/
- public enum Order {
+ enum Order {
DESCENDING,
ASCENDING
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
index 6bc8b1b..1687162 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
@@ -51,21 +51,39 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
* @param order The order order. Descending is most efficient
* @param last The value to start seeking from. Must be >= this value
* @param order
+ *
+ * //TODO, make last an optional
*/
- public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order, final Edge last
- ) {
+ public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order, final Edge last ) {
+ this(node, type, maxTimestamp, order, Optional.fromNullable(last));
+ }
+
+
+ /**
+ * Create the search modules
+ * @param node The node to search from
+ * @param type The edge type
+ * @param maxTimestamp The maximum timestamp to return
+ * @param order The order order. Descending is most efficient
+ * @param last The value to start seeking from. Must be >= this value
+ * @param order
+ *
+ * //TODO, make last an optional
+ */
+ public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order, final Optional<Edge> last ) {
Preconditions.checkNotNull( order, "order is required");
- ValidationUtils.verifyIdentity(node);
+ ValidationUtils.verifyIdentity( node );
ValidationUtils.verifyString( type, "type" );
GraphValidation.validateTimestamp( maxTimestamp, "maxTimestamp" );
+ Preconditions.checkNotNull( last, "last is required" );
this.node = node;
this.type = type;
this.maxTimestamp = maxTimestamp;
this.order = order;
- this.last = Optional.fromNullable(last);
+ this.last = last;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index f167f0c..03f1df8 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -32,6 +32,7 @@ import org.apache.usergrid.persistence.core.guice.ProxyImpl;
import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
@@ -160,7 +161,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesToTarget( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, null ) );
+ new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
} );
}
@@ -177,7 +178,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesFromSource( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, null ) );
+ new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
index 6bb467f..6bf6178 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
@@ -31,6 +31,7 @@ import java.util.Map;
import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.MarkedEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.SearchByIdType;
@@ -39,6 +40,7 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
import com.netflix.astyanax.MutationBatch;
@@ -272,7 +274,8 @@ public abstract class DirectedEdgeMeta {
final Id sourceId = nodes[0].id;
final String edgeType = types[0];
- final SearchByEdgeType search = new SimpleSearchByEdgeType( sourceId, edgeType, maxValue, order, null);
+ final SearchByEdgeType search = new SimpleSearchByEdgeType( sourceId, edgeType, maxValue, order,
+ Optional.<Edge>absent());
return serialization.getEdgesFromSource( edgeColumnFamilies, scope, search, shards );
}
@@ -392,7 +395,7 @@ public abstract class DirectedEdgeMeta {
final Id targetId = nodes[0].id;
final String edgeType = types[0];
- final SearchByEdgeType search = new SimpleSearchByEdgeType( targetId, edgeType, maxValue, order, null);
+ final SearchByEdgeType search = new SimpleSearchByEdgeType( targetId, edgeType, maxValue, order, Optional.<Edge>absent());
return serialization.getEdgesToTarget( edgeColumnFamilies, scope, search, shards );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
index d3bf24e..c91fe89 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
@@ -48,6 +48,7 @@ import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import rx.Observable;
@@ -112,7 +113,8 @@ public class GraphManagerLoadTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null) );
+ return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional
+ .<Edge>absent()) );
}
};
@@ -139,7 +141,7 @@ public class GraphManagerLoadTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
+ return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
};
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index 095f855..d702556 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -60,6 +60,7 @@ import org.apache.usergrid.persistence.model.entity.Id;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Slf4jReporter;
+import com.google.common.base.Optional;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@@ -166,7 +167,7 @@ public class GraphManagerShardConsistencyIT {
public Observable<Edge> doSearch( final GraphManager manager ) {
return manager.loadEdgesFromSource(
new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
};
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
index cc131d1..0d76931 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
@@ -42,6 +42,7 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.google.common.base.Optional;
import com.google.inject.Inject;
import rx.Observable;
@@ -119,7 +120,8 @@ public class GraphManagerStressTest {
for ( Id sourceId : sourceIds ) {
final Iterable<Edge> edges = manager.loadEdgesFromSource(
- new SimpleSearchByEdgeType( sourceId, "test", timestamp, SearchByEdgeType.Order.DESCENDING, null ) )
+ new SimpleSearchByEdgeType( sourceId, "test", timestamp, SearchByEdgeType.Order.DESCENDING, Optional
+ .<Edge>absent() ) )
.toBlocking().toIterable();
for ( Edge edge : edges ) {
@@ -193,7 +195,7 @@ public class GraphManagerStressTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
+ return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
};
@@ -221,7 +223,7 @@ public class GraphManagerStressTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
+ return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
}
};
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
index a371340..39014ab 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/Id.java
@@ -22,11 +22,14 @@ package org.apache.usergrid.persistence.model.entity;
import java.io.Serializable;
import java.util.UUID;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+
/**
* Interface for creating identifiers for an entity. The implementation should implement
* the equals and hasCode methods
* @author tnine */
+@JsonDeserialize(as = SimpleId.class)
public interface Id extends Comparable<Id>, Serializable {
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 9656e2d..171baca 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -59,7 +59,7 @@ limitations under the License.
<commons.collections.version>3.2.1</commons.collections.version>
<commons.io.version>2.4</commons.io.version>
<commons.lang.version>3.1</commons.lang.version>
- <elasticsearch.version>1.3.2</elasticsearch.version>
+ <elasticsearch.version>1.4.4</elasticsearch.version>
<fasterxml-uuid.version>3.1.3</fasterxml-uuid.version>
<guava.version>18.0</guava.version>
<guice.version>4.0-beta5</guice.version>
@@ -71,7 +71,7 @@ limitations under the License.
<junit.version>4.11</junit.version>
<kryo-serializers.version>0.26</kryo-serializers.version>
<log4j.version>1.2.17</log4j.version>
- <rx.version>0.19.6</rx.version>
+ <rx.version>0.20.7</rx.version>
<slf4j.version>1.7.2</slf4j.version>
<surefire.version>2.16</surefire.version>
<aws.version>1.9.0</aws.version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 9a7a867..e77589d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -843,6 +843,14 @@ public class Query {
}
+ /**
+ * Return true if there is no query language to execute
+ * @return
+ */
+ public boolean isGraphQuery(){
+ return rootOperand == null && ql == null;
+ }
+
@JsonIgnore
public Operand getRootOperand() {
if ( rootOperand == null ) { // attempt deserialization
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/ConnectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/ConnectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/ConnectionScenarios.scala
index aa724d6..07e11fe 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/ConnectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/ConnectionScenarios.scala
@@ -40,7 +40,7 @@ import io.gatling.http.Predef._
val postConnection = exec(
http("Connect user with device")
- .post("/${collectionName}/${entityId}/${connectionType}/${entityId}")
+ .post("/${connectionName}/${entityId}/${connectionType}/${entityId}")
.headers(Headers.jsonAuthorized)
.check(status.is(200))
)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6098d9b6/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index c458acb..9490cd7 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -125,7 +125,6 @@
<usergrid.it.threads>8</usergrid.it.threads>
<metrics.version>3.0.0</metrics.version>
- <rx.version>0.19.6</rx.version>
<surefire.plugin.artifactName>surefire-junit47</surefire.plugin.artifactName>
<surefire.plugin.version>2.18.1</surefire.plugin.version>
<powermock.version>1.6.1</powermock.version>
[07/20] incubator-usergrid git commit: Fixes empty results bug
Posted by to...@apache.org.
Fixes empty results bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6904c9db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6904c9db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6904c9db
Branch: refs/heads/two-dot-o-dev
Commit: 6904c9dbdf80e4dee303ad020d4d8c99a9f3946c
Parents: 6098d9b
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 09:35:40 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 09:35:40 2015 -0600
----------------------------------------------------------------------
.../corepersistence/results/AbstractGraphQueryExecutor.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6904c9db/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
index 0bea013..cf0764e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.corepersistence.results;
+import java.util.Collections;
import java.util.Iterator;
import java.util.NoSuchElementException;
@@ -78,7 +79,7 @@ public abstract class AbstractGraphQueryExecutor implements QueryExecutor {
//hasn't been set up yet, run through our first setup
if ( observableIterator == null ) {
- //assign them to an interator. this now uses an internal buffer with backpressure, so we won't load all
+ //assign them to an iterator. this now uses an internal buffer with backpressure, so we won't load all
// results
//set up our command builder
final CommandBuilder commandBuilder = new CommandBuilder( applicationScope, sourceId, requestCursor, limit );
@@ -91,6 +92,11 @@ public abstract class AbstractGraphQueryExecutor implements QueryExecutor {
commandBuilder.build( new EntityLoadCommand( entityCollectionManagerFactory, applicationScope ) );
this.observableIterator = resultsObservable.toBlocking().getIterator();
+
+ if(!observableIterator.hasNext()){
+ //no results, generate an empty one
+ this.observableIterator = Collections.singleton(new Results()).iterator();
+ }
}
[04/20] incubator-usergrid git commit: Merge branch 'EsTimeout' of
https://github.com/apache/incubator-usergrid into two-dot-o
Posted by to...@apache.org.
Merge branch 'EsTimeout' of https://github.com/apache/incubator-usergrid into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/472ccaf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/472ccaf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/472ccaf2
Branch: refs/heads/two-dot-o-dev
Commit: 472ccaf2523799c51a57e324776172f57ee55ce2
Parents: 527ac86 74f6684
Author: GERey <gr...@apigee.com>
Authored: Mon Apr 20 09:27:21 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Apr 20 09:27:21 2015 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 17 +++++++++++++++++
.../index/impl/EsEntityIndexImpl.java | 20 +++-----------------
.../index/impl/EsIndexBufferConsumerImpl.java | 2 +-
3 files changed, 21 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
[12/20] incubator-usergrid git commit: Refactor into commands WIP
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java
deleted file mode 100644
index b74f433..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionRefsVerifier.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public class CollectionRefsVerifier extends VersionVerifier {
-
-
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
- List<EntityRef> refs = new ArrayList<EntityRef>(ids.size());
- for ( Id id : ids ) {
- refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
- }
- return Results.fromRefList( refs );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
deleted file mode 100644
index 08458c3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.SearchEdge;
-
-
-/**
- * Factory for creating results
- */
-public class CollectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
-
- private final ManagerCache managerCache;
-
-
- public CollectionResultsLoaderFactoryImpl( final ManagerCache managerCache ) {
- this.managerCache = managerCache;
- }
-
-
- @Override
- public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope, final Query.Level resultsLevel ) {
-
- ResultsVerifier verifier;
-
- if ( resultsLevel == Query.Level.REFS ) {
- verifier = new CollectionRefsVerifier();
- }
- else if ( resultsLevel == Query.Level.IDS ) {
- verifier = new IdsVerifier();
- }
- else {
- verifier = new EntityVerifier(Query.MAX_LIMIT);
- }
-
- return new FilteringLoader( managerCache, verifier, applicationScope, scope );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java
deleted file mode 100644
index 408edd3..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionRefsVerifier.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.usergrid.persistence.ConnectedEntityRef;
-import org.apache.usergrid.persistence.ConnectionRef;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.cassandra.ConnectedEntityRefImpl;
-import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
-
-
-/**
- * Verifier for creating connections
- */
-public class ConnectionRefsVerifier extends VersionVerifier {
-
-
- private final EntityRef ownerId;
- private final String connectionType;
-
-
- public ConnectionRefsVerifier( final EntityRef ownerId, final String connectionType ) {
- this.ownerId = ownerId;
- this.connectionType = connectionType;
- }
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
- List<ConnectionRef> refs = new ArrayList<>();
- for ( Id id : ids ) {
- refs.add( new ConnectionRefImpl( ownerId, connectionType, ref(id.getType(), id.getUuid()) ));
- }
-
- return Results.fromConnections( refs );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java
deleted file mode 100644
index e9633c9..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionResultsLoaderFactoryImpl.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.SearchEdge;
-
-
-/**
- * Factory for creating results
- */
-public class ConnectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
-
- private final ManagerCache managerCache;
- private final EntityRef ownerId;
- private final String connectionType;
-
-
- public ConnectionResultsLoaderFactoryImpl( final ManagerCache managerCache, final EntityRef ownerId,
- final String connectionType ) {
- this.managerCache = managerCache;
- this.ownerId = ownerId;
- this.connectionType = connectionType;
- }
-
-
- @Override
- public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope, final Query.Level resultsLevel ) {
-
- ResultsVerifier verifier;
-
- if ( resultsLevel == Query.Level.REFS ) {
- verifier = new ConnectionRefsVerifier( ownerId, connectionType );
- }
- else if ( resultsLevel == Query.Level.IDS ) {
- verifier = new ConnectionRefsVerifier( ownerId, connectionType );;
- }
- else {
- verifier = new EntityVerifier(Query.MAX_LIMIT);
- }
-
- return new FilteringLoader( managerCache, verifier, applicationScope, scope );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
deleted file mode 100644
index 4a1e43a..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ElasticSearchQueryExecutor.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.usergrid.persistence.index.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import com.google.common.base.Optional;
-
-
-public class ElasticSearchQueryExecutor implements QueryExecutor {
-
- private static final Logger logger = LoggerFactory.getLogger( ElasticSearchQueryExecutor.class );
-
- private final ResultsLoaderFactory resultsLoaderFactory;
-
- private final ApplicationScope applicationScope;
-
- private final ApplicationEntityIndex entityIndex;
-
- private final SearchEdge indexScope;
-
- private final SearchTypes types;
-
- private final Query query;
-
-
- private Results currentResults;
-
- private boolean moreToLoad = true;
-
-
- public ElasticSearchQueryExecutor( final ResultsLoaderFactory resultsLoaderFactory, final ApplicationEntityIndex entityIndex,
- final ApplicationScope applicationScope, final SearchEdge indexScope,
- final SearchTypes types, final Query query ) {
- this.resultsLoaderFactory = resultsLoaderFactory;
- this.applicationScope = applicationScope;
- this.entityIndex = entityIndex;
- this.indexScope = indexScope;
- this.types = types;
-
- //we must deep copy the query passed. Otherwise we will modify it's state with cursors. Won't fix, not relevant
- //once we start subscribing to streams.
- this.query = new Query(query);
- }
-
-
- @Override
- public Iterator<Results> iterator() {
- return this;
- }
-
-
- private void loadNextPage() {
- // Because of possible stale entities, which are filtered out by buildResults(),
- // we loop until the we've got enough results to satisfy the query limit.
-
- final int maxQueries = 10; // max re-queries to satisfy query limit
-
- final int originalLimit = query.getLimit();
-
- Results results = null;
- int queryCount = 0;
-
-
- CandidateResults crs = null;
-
- while ( queryCount++ < maxQueries ) {
-
- crs = getCandidateResults( query );
-
-
- logger.debug( "Calling build results with crs {}", crs );
- results = buildResults( indexScope, query, crs );
-
- /**
- * In an edge case where we delete stale entities, we could potentially get less results than expected.
- * This will only occur once during the repair phase.
- * We need to ensure that we short circuit before we overflow the requested limit during a repair.
- */
- if ( crs.isEmpty() || !crs.hasOffset() || results.size() > 0 ) { // no results, no cursor, can't get more
- break;
- }
-
-
- //we didn't load anything, but there was a cursor, this means a read repair occured. We have to short
- //circuit to avoid over returning the result set
-
-
- // need to query for more
- // ask for just what we need to satisfy, don't want to exceed limit
- query.setOffsetFromCursor(results.getCursor());
- query.setLimit( originalLimit - results.size() );
-
- logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
- originalLimit, query.getLimit(), queryCount
- } );
- }
-
- //now set our cursor if we have one for the next iteration
- if ( results.hasCursor() ) {
- query.setOffsetFromCursor(results.getCursor());
- moreToLoad = true;
- }
-
- else {
- moreToLoad = false;
- }
-
-
- //set our select subjects into our query if provided
- if(crs != null){
- query.setSelectSubjects( crs.getGetFieldMappings() );
- }
-
-
- //set our current results and the flag
- this.currentResults = results;
- }
-
-
- /**
- * Get the candidates or load the cursor, whichever we require
- * @param query
- * @return
- */
- private CandidateResults getCandidateResults(final Query query){
- final Optional<Integer> cursor = query.getOffset();
- final String queryToExecute = query.getQl().or("select *");
-
- CandidateResults results = cursor.isPresent()
- ? entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get())
- : entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
-
- return results;
- }
-
-
- /**
- * Build results from a set of candidates, and discard those that represent stale indexes.
- *
- * @param query Query that was executed
- * @param crs Candidates to be considered for results
- */
- private Results buildResults( final SearchEdge indexScope, final Query query, final CandidateResults crs ) {
-
- logger.debug( "buildResults() from {} candidates", crs.size() );
-
- //get an instance of our results loader
- final ResultsLoader resultsLoader =
- this.resultsLoaderFactory.getLoader( applicationScope, indexScope, query.getResultsLevel() );
-
- //load the results
- final Results results = resultsLoader.loadResults(crs);
-
- //signal for post processing
- resultsLoader.postProcess();
-
- //set offset into query
- if(crs.getOffset().isPresent()) {
- query.setOffset(crs.getOffset().get());
- }else{
- query.clearOffset();
- }
- results.setCursorFromOffset( query.getOffset() );
-
- //ugly and tight coupling, but we don't have a choice until we finish some refactoring
- results.setQueryExecutor( this );
-
- logger.debug( "Returning results size {}", results.size() );
-
- return results;
- }
-
-
- @Override
- public boolean hasNext() {
-
- //we've tried to load and it's empty and we have more to load, load the next page
- if ( currentResults == null ) {
- //there's nothing left to load, nothing to do
- if ( !moreToLoad ) {
- return false;
- }
-
- //load the page
-
- loadNextPage();
- }
-
-
- //see if our current results are not null
- return currentResults != null;
- }
-
-
- @Override
- public Results next() {
- if ( !hasNext() ) {
- throw new NoSuchElementException( "No more results present" );
- }
-
- final Results toReturn = currentResults;
-
- currentResults = null;
-
- return toReturn;
- }
-
- @Override
- public void remove() {
- throw new RuntimeException("Remove not implemented!!");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
deleted file mode 100644
index a2dd5e0..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-import com.google.common.base.Optional;
-
-
-/**
- * A loader that verifies versions are correct in cassandra and match elasticsearch
- */
-public class EntityVerifier implements ResultsVerifier {
-
- private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
-
- private EntitySet ids;
-
- private Map<Id, org.apache.usergrid.persistence.model.entity.Entity> entityMapping;
-
-
- public EntityVerifier( final int maxSize ) {
- this.entityMapping = new HashMap<>( maxSize );
- }
-
-
- @Override
- public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
- ids = ecm.load( idsToLoad ).toBlocking().last();
- logger.debug("loadResults() asked for {} ids and got {}", idsToLoad.size(), ids.size());
- }
-
-
- @Override
- public boolean isValid( final CandidateResult candidateResult ) {
- final Id entityId = candidateResult.getId();
-
- final MvccEntity savedEntity = ids.getEntity( entityId );
-
- //version wasn't found deindex
- if ( savedEntity == null ) {
- logger.warn( "Version for Entity {}:{} not found", entityId.getType(), entityId.getUuid() );
- return false;
- }
-
- final UUID candidateVersion = candidateResult.getVersion();
- final UUID savedVersion = savedEntity.getVersion();
-
- if ( UUIDComparator.staticCompare( savedVersion, candidateVersion ) > 0 ) {
- logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
- entityId.getUuid(), entityId.getType(), candidateVersion, savedEntity
- } );
-
- return false;
- }
-
-
- final Optional<org.apache.usergrid.persistence.model.entity.Entity> entity = savedEntity.getEntity();
-
- if ( !entity.isPresent() ) {
- logger.warn( "Entity uuid:{} version v:{} is deleted but indexed, this is a bug ",
- entityId.getUuid(), savedEntity.getEntity() );
- return false;
- }
-
- entityMapping.put( entityId, entity.get() );
-
- return true;
- }
-
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
-
- final List<Entity> ugEntities = new ArrayList<>( ids.size() );
-
- for ( final Id id : ids ) {
- final org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityMapping.get( id );
-
- Entity entity = EntityFactory.newEntity( id.getUuid(), id.getType() );
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
- entity.addProperties( entityMap );
- ugEntities.add( entity );
- }
-
- return Results.fromEntities( ugEntities );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
deleted file mode 100644
index a48e1b8..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-
-import javax.annotation.Nullable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.index.CandidateResults;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-
-
-public class FilteringLoader implements ResultsLoader {
-
- private static final Logger logger = LoggerFactory.getLogger( FilteringLoader.class );
-
- private final ManagerCache managerCache;
- private final ResultsVerifier resultsVerifier;
- private final ApplicationScope applicationScope;
- private final SearchEdge indexScope;
- private final EntityIndexBatch indexBatch;
-
-
- /**
- * Create an instance of a filter loader
- *
- * @param managerCache The manager cache to load
- * @param resultsVerifier The verifier to verify the candidate results
- * @param applicationScope The application scope to perform the load
- * @param indexScope The index scope used in the search
- */
- protected FilteringLoader( final ManagerCache managerCache, final ResultsVerifier resultsVerifier,
- final ApplicationScope applicationScope, final SearchEdge indexScope ) {
-
- this.managerCache = managerCache;
- this.resultsVerifier = resultsVerifier;
- this.applicationScope = applicationScope;
- this.indexScope = indexScope;
-
- final ApplicationEntityIndex index = managerCache.getEntityIndex( applicationScope );
-
- indexBatch = index.createBatch();
- }
-
-
- @Override
- public Results loadResults( final CandidateResults crs ) {
-
-
- if ( crs.size() == 0 ) {
- return new Results();
- }
-
-
- // For each entity, holds the index it appears in our candidates for keeping ordering correct
- final Map<Id, Integer> orderIndex = new HashMap<>( crs.size() );
-
- // Maps the entity ids to our candidates
- final Map<Id, CandidateResult> maxCandidateMapping = new HashMap<>( crs.size() );
-
-
- final Iterator<CandidateResult> iter = crs.iterator();
-
-
- // TODO, in this case we're "optimizing" due to the limitations of collection scope.
- // Perhaps we should change the API to just be an application, then an "owner" scope?
-
- // Go through the candidates and group them by scope for more efficient retrieval.
- // Also remove duplicates before we even make a network call
- for ( int i = 0; iter.hasNext(); i++ ) {
-
- final CandidateResult currentCandidate = iter.next();
-
- final Id entityId = currentCandidate.getId();
-
- //check if we've seen this candidate by id
- final CandidateResult previousMax = maxCandidateMapping.get( entityId );
-
- //its not been seen, save it
- if ( previousMax == null ) {
- maxCandidateMapping.put( entityId, currentCandidate );
- orderIndex.put( entityId, i );
- continue;
- }
-
- //we have seen it, compare them
-
- final UUID previousMaxVersion = previousMax.getVersion();
-
- final UUID currentVersion = currentCandidate.getVersion();
-
-
- final CandidateResult toRemove;
- final CandidateResult toKeep;
-
- //current is newer than previous. Remove previous and keep current
- if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
- toRemove = previousMax;
- toKeep = currentCandidate;
- }
- //previously seen value is newer than current. Remove the current and keep the previously seen value
- else {
- toRemove = currentCandidate;
- toKeep = previousMax;
- }
-
- //this is a newer version, we know we already have a stale entity, add it to be cleaned up
-
-
- //de-index it
- logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
- entityId.getUuid(), entityId.getType(), toRemove.getVersion(), toKeep.getVersion()
- } );
-
- //deindex this document, and remove the previous maxVersion
- //we have to deindex this from our ownerId, since this is what gave us the reference
- indexBatch.deindex( indexScope, toRemove );
-
-
- //TODO, fire the entity repair cleanup task here instead of de-indexing
-
- //replace the value with a more current version
- maxCandidateMapping.put( entityId, toKeep );
- orderIndex.put( entityId, i );
- }
-
-
- //now everything is ordered, and older versions are removed. Batch fetch versions to verify
- // existence and correct versions
-
- final TreeMap<Integer, Id> sortedResults = new TreeMap<>();
-
-
- final Collection<Id> idsToLoad =
- Collections2.transform( maxCandidateMapping.values(), new Function<CandidateResult, Id>() {
- @Nullable
- @Override
- public Id apply( @Nullable final CandidateResult input ) {
- //NOTE this is never null, we won't need to check
- return input.getId();
- }
- } );
-
-
- //now using the scope, load the collection
-
- // Get the collection scope and batch load all the versions. We put all entities in
- // app/app for easy retrieval/ unless persistence changes, we never want to read from
- // any scope other than the app, app, scope name scope
- // final CollectionScope collScope = new CollectionScopeImpl(
- // applicationScope.getApplication(), applicationScope.getApplication(), scopeName);
-
- final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( applicationScope );
-
-
- //load the results into the loader for this scope for validation
- resultsVerifier.loadResults( idsToLoad, ecm );
-
- //now let the loader validate each candidate. For instance, the "max" in this candidate
- //could still be a stale result, so it needs validated
- for ( final Id requestedId : idsToLoad ) {
-
- final CandidateResult cr = maxCandidateMapping.get( requestedId );
-
- //ask the loader if this is valid, if not discard it and de-index it
- if ( !resultsVerifier.isValid( cr ) ) {
- indexBatch.deindex( indexScope, cr );
- continue;
- }
-
- //if we get here we're good, we need to add this to our results
- final int candidateIndex = orderIndex.get( requestedId );
-
- sortedResults.put( candidateIndex, requestedId );
- }
-
-
- // NOTE DO NOT execute the batch here.
- // It changes the results and we need consistent paging until we aggregate all results
- return resultsVerifier.getResults( sortedResults.values() );
- }
-
-
- @Override
- public void postProcess() {
- this.indexBatch.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java
deleted file mode 100644
index fae6eb1..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public class IdsVerifier extends VersionVerifier {
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
-
- final List<UUID> returnIds = new ArrayList<>( ids.size() );
-
- for ( final Id id : ids ) {
- returnIds.add( id.getUuid() );
- }
-
-
- return Results.fromIdList( returnIds );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
new file mode 100644
index 0000000..382fb64
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ObservableQueryExecutor.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.results;
+
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.usergrid.persistence.Results;
+
+import rx.Observable;
+
+
+/**
+ * Our proxy to allow us to subscribe to observable results, then return them as an interator. A bridge for 2.0 -> 1.0
+ * code
+ */
+public class ObservableQueryExecutor implements QueryExecutor {
+
+ private final Observable<Results> resultsObservable;
+
+ public Iterator<Results> iterator;
+
+
+ public ObservableQueryExecutor( final Observable<Results> resultsObservable ) {
+ this.resultsObservable = resultsObservable;
+ }
+
+
+ @Override
+ public Iterator<Results> iterator() {
+ return this;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+
+ if ( iterator == null ) {
+ iterator = resultsObservable.toBlocking().getIterator();
+ }
+
+
+ return iterator.hasNext();
+ }
+
+
+ @Override
+ public Results next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more results present" );
+ }
+ final Results next = iterator.next();
+
+ next.setQueryExecutor( this );
+
+ return next;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
deleted file mode 100644
index e8a7bdf..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.index.CandidateResults;
-
-
-/**
- * Interface for loading results
- */
-public interface ResultsLoader {
-
- /**
- * Using the candidate results, load our results. Should filter stale results
- * @param crs The candidate result set
- * @return Results. Null safe, but may be empty
- */
- public Results loadResults( final CandidateResults crs);
-
- /**
- * Post process the load operation
- */
- public void postProcess();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
deleted file mode 100644
index 3399a35..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.SearchEdge;
-
-
-/**
- * Factory for creating results
- */
-public interface ResultsLoaderFactory {
-
- /**
- * Get the loader for results
- * @param applicationScope The application scope used to load results
- * @param indexScope The index scope used in the search
- * @param
- */
- ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge indexScope,
- final Query.Level resultsLevel );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java
deleted file mode 100644
index 46e0983..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.Collection;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public interface ResultsVerifier {
-
- /**
- * Load all the candidate ides for verification
- * @param ids The Id's to load
- * @param ecm The entity collection manager
- */
- public void loadResults(Collection<Id> ids, EntityCollectionManager ecm);
-
- /**
- * Return true if the candidate result is a valid result that should be retained. If it should
- * not it should also be removed from the list of possible return values in this loader
- * @param candidateResult
- */
- public boolean isValid(CandidateResult candidateResult);
-
-
- /**
- * Load the result set with the given ids
- * @return
- */
- public Results getResults(Collection<Id> ids);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java
deleted file mode 100644
index 574ba55..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.results;
-
-
-import java.util.Collection;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.VersionSet;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-
-
-/**
- * A loader that verifies versions are correct in Cassandra and match ElasticSearch
- */
-public abstract class VersionVerifier implements ResultsVerifier {
-
- private static final Logger logger = LoggerFactory.getLogger( VersionVerifier.class );
-
- private VersionSet ids;
-
-
- @Override
- public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
- ids = ecm.getLatestVersion( idsToLoad ).toBlocking().last();
- }
-
-
- @Override
- public boolean isValid( final CandidateResult candidateResult ) {
- final Id entityId = candidateResult.getId();
-
- final MvccLogEntry version = ids.getMaxVersion( entityId );
-
- //version wasn't found ,deindex
- if ( version == null ) {
- logger.warn( "Version for Entity {}:{} not found",
- entityId.getUuid(), entityId.getUuid() );
-
- return false;
- }
-
- final UUID savedVersion = version.getVersion();
-
- if ( UUIDComparator.staticCompare( savedVersion, candidateResult.getVersion() ) > 0 ) {
- logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] {
- entityId.getUuid(),
- entityId.getType(),
- candidateResult.getVersion(),
- savedVersion
- } );
-
- return false;
- }
-
- return true;
- }
-
-}
[15/20] incubator-usergrid git commit: First pass at new command
layout. Need to test and debug
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
deleted file mode 100644
index 58e4296..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.Collection;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.VersionSet;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-
-
-/**
- * A loader that verifies versions are correct in Cassandra and match ElasticSearch
- */
-public abstract class VersionVerifier implements ResultsVerifier {
-
- private static final Logger logger = LoggerFactory.getLogger( VersionVerifier.class );
-
- private VersionSet ids;
-
-
- @Override
- public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
- ids = ecm.getLatestVersion( idsToLoad ).toBlocking().last();
- }
-
-
- @Override
- public boolean isValid( final CandidateResult candidateResult ) {
- final Id entityId = candidateResult.getId();
-
- final MvccLogEntry version = ids.getMaxVersion( entityId );
-
- //version wasn't found ,deindex
- if ( version == null ) {
- logger.warn( "Version for Entity {}:{} not found",
- entityId.getUuid(), entityId.getUuid() );
-
- return false;
- }
-
- final UUID savedVersion = version.getVersion();
-
- if ( UUIDComparator.staticCompare( savedVersion, candidateResult.getVersion() ) > 0 ) {
- logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] {
- entityId.getUuid(),
- entityId.getType(),
- candidateResult.getVersion(),
- savedVersion
- } );
-
- return false;
- }
-
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
deleted file mode 100644
index 1504f26..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.entity;
-
-
-import java.io.Serializable;
-import java.util.Map;
-
-import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
-import org.apache.usergrid.corepersistence.command.read.Collector;
-import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Loads entities from a set of Ids.
- *
- * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
- */
-public class EntityLoadCollector extends AbstractCommand<Results, Serializable> implements Collector<Results> {
-
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
-
- //TODO get rid of this when merged into 2.0 dev
- private final ApplicationScope applicationScope;
- private int resultSize;
-
-
- public EntityLoadCollector( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final ApplicationScope applicationScope ) {
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.applicationScope = applicationScope;
- }
-
-
- @Override
- protected Class<Serializable> getCursorClass() {
- return null;
- }
-
-
- @Override
- public Observable<Results> call( final Observable<Id> observable ) {
-
-
- /**
- * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
- * objects
- */
-
- final EntityCollectionManager entityCollectionManager =
- entityCollectionManagerFactory.createCollectionManager( applicationScope );
-
- final Observable<EntitySet> entitySetObservable = observable.buffer( resultSize ).flatMap(
- bufferedIds -> Observable.just( bufferedIds ).flatMap( ids -> entityCollectionManager.load( ids ) ) );
-
-
- final Observable<Results> resultsObservable = entitySetObservable
-
- .flatMap( entitySet -> {
-
- //get our entites and filter missing ones, then collect them into a results object
- final Observable<MvccEntity> mvccEntityObservable = Observable.from( entitySet.getEntities() );
-
- //convert them to our old entity model, then filter nulls, meaning they weren't found
- return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter( entity -> entity == null )
-
- //convert them to a list, then map them into results
- .toList().map( entities -> {
- final Results results = Results.fromEntities( entities );
- results.setCursor( generateCursor() );
-
- return results;
- } )
- //if no results are present, return an empty results
- .singleOrDefault( new Results( ) );
- } );
-
-
- return resultsObservable;
- }
-
- /**
- * Map a new cp entity to an old entity. May be null if not present
- */
-
-
- private org.apache.usergrid.persistence.Entity mapEntity( final MvccEntity mvccEntity ) {
- if ( !mvccEntity.getEntity().isPresent() ) {
- return null;
- }
-
-
- final Entity cpEntity = mvccEntity.getEntity().get();
- final Id entityId = cpEntity.getId();
-
- org.apache.usergrid.persistence.Entity entity =
- EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
- entity.addProperties( entityMap );
-
- return entity;
- }
-
-
- @Override
- public void setLimit( final int limit ) {
- this.resultSize = limit;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
deleted file mode 100644
index 95b6c6b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.graph;
-
-
-import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
-import org.apache.usergrid.corepersistence.command.read.TraverseCommand;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
-
-
-/**
- * Command for reading graph edges
- */
-public abstract class AbstractReadGraphCommand extends AbstractCommand<Id, Edge> implements TraverseCommand {
-
- private final GraphManagerFactory graphManagerFactory;
-
-
- /**
- * Create a new instance of our command
- * @param graphManagerFactory
- */
- public AbstractReadGraphCommand( final GraphManagerFactory graphManagerFactory ) {
- this.graphManagerFactory = graphManagerFactory;
- }
-
-
- @Override
- public Observable<Id> call( final Observable<Id> observable ) {
-
- //get the graph manager
- final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
-
- //set our our constant state
- final Optional<Edge> startFromCursor = getCursor();
-
- final String edgeName = getEdgeTypeName();
-
-
- //return all ids that are emitted from this edge
- return observable.flatMap( new Func1<Id, Observable<Id>>() {
-
- @Override
- public Observable<Id> call( final Id id ) {
-
- final SimpleSearchByEdgeType search = new SimpleSearchByEdgeType(id,edgeName, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, startFromCursor );
-
- /**
- * TODO, pass a message with pointers to our cursor values to be generated later
- */
- return graphManager.loadEdgesFromSource( search ).doOnNext( new Action1<Edge>() {
- @Override
- public void call( final Edge edge ) {
- setCursor( edge );
- }
- } ).map( new Func1<Edge, Id>() {
- @Override
- public Id call( final Edge edge ) {
- return edge.getTargetNode();
- }
- } );
- }
- } );
- }
-
-
- @Override
- protected Class<Edge> getCursorClass() {
- return Edge.class;
- }
-
-
-
- /**
- * Get the edge type name we should use when traversing
- * @return
- */
- protected abstract String getEdgeTypeName();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
deleted file mode 100644
index 3b6633b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.graph;
-
-
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.inject.Inject;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
-
-
-/**
- * Command for reading graph edges on a collection
- */
-public class ReadGraphCollectionCommand extends AbstractReadGraphCommand {
-
- private final String collectionName;
-
-
- /**
- * Create a new instance of our command
- */
- @Inject
- public ReadGraphCollectionCommand( final GraphManagerFactory graphManagerFactory, final String collectionName ) {
- super( graphManagerFactory );
- this.collectionName = collectionName;
- }
-
-
- @Override
- protected String getEdgeTypeName() {
- return getCollectionScopeNameFromCollectionName( collectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
deleted file mode 100644
index 26b30e7..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.graph;
-
-
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.inject.Inject;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
-
-
-/**
- * Command for reading graph edges on a connection
- */
-public class ReadGraphConnectionCommand extends AbstractReadGraphCommand {
-
- private final String connectionName;
-
-
- /**
- * Create a new instance of our command
- */
- @Inject
- public ReadGraphConnectionCommand( final GraphManagerFactory graphManagerFactory, final String connectionName ) {
- super( graphManagerFactory );
- this.connectionName = connectionName;
- }
-
-
- @Override
- protected String getEdgeTypeName() {
- return getConnectionScopeName( connectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java
new file mode 100644
index 0000000..8463df9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.Filter;
+import org.apache.usergrid.corepersistence.pipeline.read.TraverseFilter;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * A pipeline that will allow us to build a traversal command for execution
+ *
+ * See http://martinfowler.com/articles/collection-pipeline/ for some examples
+ *
+ * TODO: Re work the cursor and limit phases. They need to be lazily evaluated, not added on build time
+ */
+public class DataPipeline {
+
+
+ private final ApplicationScope applicationScope;
+ private final List<TraverseFilter> traverseFilterList;
+
+ private Optional<String> cursor;
+ private int limit;
+
+
+ private int count = 0;
+
+
+ /**
+ * Our first pass, where we implement our start point as an Id until we can use this to perform our entire
+ * traversal. Eventually as we untangle the existing Query service nightmare, the sourceId will be remove and
+ * should only be traversed from the root application
+ */
+ public DataPipeline( final ApplicationScope applicationScope ) {
+
+ this.applicationScope = applicationScope;
+
+
+ traverseFilterList = new ArrayList<>();
+ }
+
+
+ /**
+ * Add a read command that will read Ids and produce Ids. This is an intermediate traversal operations
+ */
+ public DataPipeline withTraverseCommand( final TraverseFilter traverseCommand ) {
+
+ this.traverseFilterList.add( traverseCommand );
+
+ return this;
+ }
+
+
+ /**
+ * Build the final collection step, and process our filters
+ */
+ public <T> Observable<T> build( final CollectorFilter<T> pipeCollector ) {
+
+ RequestCursor requestCursor = new RequestCursor( this.cursor );
+ ResponseCursor responseCursor = new ResponseCursor();
+
+ Observable<Id> traverseObservable = Observable.just( applicationScope.getApplication() );
+
+ //build our traversal commands
+ for ( TraverseFilter filter : traverseFilterList ) {
+ setState( filter, requestCursor, responseCursor );
+
+ traverseObservable = traverseObservable.compose( filter );
+ }
+
+
+ setState( pipeCollector, requestCursor, responseCursor );
+
+ pipeCollector.setLimit( limit );
+
+ return traverseObservable.compose( pipeCollector );
+ }
+
+
+ public void setCursor( Optional<String> cursor ) {
+ this.cursor = cursor;
+ }
+
+
+ public void setLimit( final int limit ) {
+ this.limit = limit;
+ }
+
+
+ /**
+ * Set the id of the state
+ */
+ private void setState( final Filter<?> filter, final RequestCursor requestCursor,
+ final ResponseCursor responseCursor ) {
+
+ //TODO, see if we can wrap this observable in our ObservableTimer so we can see how long each filter takes
+
+
+ filter.setId( count );
+ //done for clarity
+ count++;
+
+ filter.setCursorCaches( requestCursor, responseCursor );
+ filter.setApplicationScope( applicationScope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
new file mode 100644
index 0000000..9916bc1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+
+/**
+ * Factory for creating pipeline builders
+ */
+public interface PipelineBuilderFactory {
+
+
+ /**
+ * Create a read pipeline builder
+ * @param applicationScope
+ * @return
+ */
+ ReadPipelineBuilder createReadPipelineBuilder( final ApplicationScope applicationScope );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
new file mode 100644
index 0000000..8afe2f8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+
+
+/**
+ * Configures all our pipelines
+ */
+public class PipelineModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ //Use Guice to create the builder since we don't really need to do anything
+ //other than DI when creating the filters
+ install(new FactoryModuleBuilder().build( ReadPipelineBuilder.class ));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java
new file mode 100644
index 0000000..ce2e10a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.cursor;
+
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.core.Base64Variant;
+import com.fasterxml.jackson.core.Base64Variants;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+
+
+/**
+ * A utility to serialize objects to/from cursors
+ */
+public class CursorSerializer {
+
+
+ private static final SmileFactory SMILE_FACTORY = new SmileFactory();
+
+ private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+
+ private static final Base64Variant VARIANT = Base64Variants.MODIFIED_FOR_URL;
+
+
+ /**
+ * Serialize the serializable object as a cursor
+ */
+ public static String asCursor( final Serializable cursor ) {
+
+ try {
+ return MAPPER.writer( VARIANT ).writeValueAsString( cursor );
+ }
+ catch ( JsonProcessingException e ) {
+ throw new CursorParseException( "Unable to serialize cursor", e );
+ }
+ }
+
+
+ /**
+ * Deserialize from the cursor
+ * @param cursor
+ * @return
+ * @throws CursorParseException
+ */
+ public <T extends Serializable> T fromCursor( final String cursor, final Class<T> cursorClass ) throws CursorParseException {
+ try {
+
+ final JsonParser parser = MAPPER.getFactory().createParser( cursor );
+ return MAPPER.reader( VARIANT ).readValue( parser, cursorClass);
+ }
+ catch ( Exception e ) {
+ throw new CursorParseException( "Unable to serialize cursor", e );
+ }
+ }
+
+
+ /**
+ * Thrown when we can't parse a cursor
+ */
+ public static class CursorParseException extends RuntimeException {
+ public CursorParseException( final String message, final Throwable cause ) {
+ super( message, cause );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
new file mode 100644
index 0000000..99cfd74
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.cursor;
+
+
+import java.io.Serializable;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A cursor that has been passed in with our request. Adds utils for parsing values
+ */
+public class RequestCursor {
+
+ public RequestCursor(final Optional<String> cursor){
+
+ }
+
+
+ /**
+ * Get the cursor with the specified id
+ */
+ public <T extends Serializable> T getCursor( final int id, final Class<T> cursorType ) {
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
new file mode 100644
index 0000000..0e047ed
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.cursor;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * A cursor used in rendering a response
+ */
+public class ResponseCursor {
+
+
+
+ /**
+ * We use a map b/c some indexes might be skipped
+ */
+ private Map<Integer, ? super Serializable> cursors = new HashMap<>();
+
+ /**
+ * Set the possible cursor value into the index. DOES NOT parse the cursor. This is intentional for performance
+ */
+ public <T extends Serializable> void setCursor( final int id, final T cursor ) {
+ cursors.put( id, cursor );
+ }
+
+
+ private void ensureCapacity() {
+
+ }
+
+
+ public String encodeAsString() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
new file mode 100644
index 0000000..a6e7ef1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import java.io.Serializable;
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * Basic functionality for our commands to handle cursor IO
+ */
+public abstract class AbstractFilter<T, C extends Serializable> implements Filter<T> {
+
+ private int id;
+ /**
+ * The cache of the cursor that was set when the read was started
+ */
+ private RequestCursor readCache;
+
+ /**
+ * The current state of the write cache. Gets updated as we traverse the observables
+ */
+ private ResponseCursor writeCache;
+
+
+ /**
+ * The applicationScope
+ */
+ protected ApplicationScope applicationScope;
+
+
+ @Override
+ public void setId( final int id ) {
+ this.id = id;
+ }
+
+
+ @Override
+ public void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache ) {
+ this.readCache = readCache;
+ this.writeCache = writeCache;
+ }
+
+
+ @Override
+ public void setApplicationScope( final ApplicationScope applicationScope ) {
+ this.applicationScope = applicationScope;
+ }
+
+
+ /**
+ * Return the parsed value of the cursor from the last request, if it exists
+ */
+ protected Optional<C> getCursor() {
+ final C cursor = readCache.getCursor( id, getCursorClass() );
+
+ return Optional.fromNullable( cursor );
+ }
+
+
+
+
+
+ /**
+ * Set the cursor value into the new cursor write cache
+ * @param newValue
+ */
+ protected void setCursor(final C newValue){
+ writeCache.setCursor( id, newValue );
+ }
+
+
+ /**
+ * Generate our state as a cursor
+ * @return
+ */
+ protected String generateCursor(){
+ return writeCache.encodeAsString();
+ }
+
+ /**
+ * Return the class to be used when parsing the cursor
+ */
+ protected abstract Class<C> getCursorClass();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java
new file mode 100644
index 0000000..883e910
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+/**
+ * A command that is used to reduce our stream of results into a final output
+ * @param <T>
+ */
+public interface CollectorFilter<T> extends Filter<T> {
+
+ /**
+ * Set the prefered result size for the command
+ * @param limit
+ */
+ void setLimit( final int limit );
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
new file mode 100644
index 0000000..670d2ed
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Interface for filtering commands. All filters must take an observable of Id's as an input. Output is then determined by subclasses.
+ * This takes an input of Id, performs some operation, and emits values for further processing in the Observable
+ * pipeline
+ */
+public interface Filter<T> extends Observable.Transformer<Id, T> {
+
+
+ /**
+ * Set the id of this command in it's execution environment
+ */
+ void setId( final int id );
+
+ /**
+ * Set the cursor cache into the command
+ *
+ * @param readCache Set the cache that was used in the request
+ * @param writeCache Set the cache to be used when writing the results
+ */
+ void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache );
+
+ /**
+ * Set the application scope of the command
+ * @param applicationScope
+ */
+ void setApplicationScope(final ApplicationScope applicationScope);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
new file mode 100644
index 0000000..525678c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryCollectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryConnectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByTypeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A factory for generating read commands
+ */
+public interface FilterFactory {
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ ReadGraphCollectionFilter readGraphCollectionCommand( final String collectionName );
+
+ /**
+ * Read a connection between two entities, the incoming and the target entity
+ * @param collectionName
+ * @param targetId
+ * @return
+ */
+ ReadGraphCollectionByIdFilter readGraphCollectionByIdFilter(final String collectionName, final Id targetId);
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ ReadGraphConnectionFilter readGraphConnectionCommand( final String connectionName );
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ ReadGraphConnectionByTypeFilter readGraphConnectionCommand( final String connectionName, final String entityType );
+
+
+ /**
+ * Read a connection directly between two identifiers
+ * @param connectionName
+ * @param targetId
+ * @return
+ */
+ ReadGraphConnectionByIdFilter readGraphConnectionByIdFilter(final String connectionName, final Id targetId);
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ EntityLoadCollectorFilter entityLoadCollector();
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector( final String collectionName, final String query,
+ final Optional<String> cursor, final int limit );
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName,final String query,
+ final Optional<String> cursor, final int limit );
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ */
+ QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName, final String connectionEntityType, final String query,
+ final Optional<String> cursor, final int limit );
+
+
+ /**
+ * Get an entity id filter. Used as a 1.0->2.0 bridge since we're not doing full traversals
+ */
+ EntityIdFilter getEntityIdFilter( final Id entityId );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
new file mode 100644
index 0000000..13aa3eb
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * An instance of a pipeline builder for building commands.
+ * Each invocation of the method will assemple the underlying pipe and updating it's state
+ * Results are added by invoking execute.
+ */
+public interface ReadPipelineBuilder {
+
+ /**
+ * An operation to bridge 2.0-> 1.0. Should be removed when everyone uses the pipeline
+ * @param id
+ * @return
+ */
+ ReadPipelineBuilder setStartId(final Id id);
+
+ /**
+ * Add a get entity to the pipeline
+ */
+ ReadPipelineBuilder getEntityViaCollection( final String collectionName, final Id entityId );
+
+
+ /**
+ * Add get Collection from our previous source
+ */
+ ReadPipelineBuilder getCollection( final String collectionName );
+
+ /**
+ * Get all entities with a query
+ */
+ ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query, final int limit,
+ final Optional<String> cursor );
+
+ /**
+ * Get an entity via the connection name and entity Id
+ */
+ ReadPipelineBuilder getEntityViaConnection( final String connectionName, final Id entityId );
+
+ /**
+ * Get all entities in a connection by the connection name
+ */
+ ReadPipelineBuilder getConnection( final String connectionName );
+
+ /**
+ * Get all entities in a connection of the specified connection type
+ */
+ ReadPipelineBuilder getConnection( final String connectionName, final String entityType );
+
+ /**
+ * Get all entities in a connection with a query
+ */
+ ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query, final int limit,
+ final Optional<String> cursor );
+
+
+ /**
+ * Get all entities in a connection with a query
+ */
+ ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType, final String query, final int limit,
+ final Optional<String> cursor );
+
+
+
+
+
+
+ /**
+ * Execute final construction of the pipeline and return the results
+ * @return
+ */
+ Observable<Results> execute();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
new file mode 100644
index 0000000..a44f4b8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import java.util.Collections;
+import java.util.UUID;
+
+import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+
+/**
+ * An implementation of our builder for piplines
+ */
+public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
+
+
+ private final FilterFactory filterFactory;
+
+ private final DataPipeline pipeline;
+
+ /**
+ * Our pointer to our collect filter. Set or cleared with each operation that's performed so the correct
+ * results are rendered
+ */
+ private CollectorFilter<Results> collectorFilter;
+
+
+ @Inject
+ public ReadPipelineBuilderImpl( final FilterFactory filterFactory,
+ @Assisted final ApplicationScope applicationScope ) {
+ this.filterFactory = filterFactory;
+ this.pipeline = new DataPipeline( applicationScope );
+ }
+
+
+ @Override
+ public ReadPipelineBuilder setStartId( final Id id ) {
+ pipeline.withTraverseCommand( filterFactory.getEntityIdFilter( id ) );
+
+ this.collectorFilter = null;
+
+
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder getEntityViaCollection( final String collectionName, final Id entityId ) {
+
+ pipeline.withTraverseCommand( filterFactory.readGraphCollectionByIdFilter( collectionName, entityId ) );
+
+ setEntityLoaderFilter();
+
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder getCollection( final String collectionName ) {
+
+
+ pipeline.withTraverseCommand( filterFactory.readGraphCollectionCommand( collectionName ) );
+
+ setEntityLoaderFilter();
+
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query, final int limit,
+ final Optional<String> cursor ) {
+
+ //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+ collectorFilter = filterFactory.queryCollectionElasticSearchCollector( collectionName, query, cursor, limit );
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder getEntityViaConnection( final String connectionName, final Id entityId ) {
+
+ pipeline.withTraverseCommand( filterFactory.readGraphConnectionByIdFilter( connectionName, entityId ) );
+ setEntityLoaderFilter();
+
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder getConnection( final String connectionName ) {
+
+ pipeline.withTraverseCommand( filterFactory.readGraphConnectionCommand( connectionName ) );
+ setEntityLoaderFilter();
+
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder getConnection( final String connectionName, final String entityType ) {
+ pipeline.withTraverseCommand( filterFactory.readGraphConnectionCommand( connectionName, entityType ) );
+ setEntityLoaderFilter();
+
+ return this;
+ }
+
+ /**
+ *
+ * @param connectionName
+ * @param query
+ * @param limit
+ * @param cursor
+ * @return
+ */
+ @Override
+ public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query, final int limit,
+ final Optional<String> cursor ) {
+
+ //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+ collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, query, cursor, limit );
+
+ return this;
+ }
+
+
+ @Override
+ public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType,
+ final String query, final int limit,
+ final Optional<String> cursor ) {
+
+ //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+ collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, entityType, query, cursor, limit );
+ return this;
+ }
+
+
+
+
+ @Override
+ public Observable<Results> execute() {
+ Preconditions.checkNotNull(collectorFilter, "You have not specified an operation that creates a collection filter. This is required for loading results");
+
+ return pipeline.build( collectorFilter );
+ }
+
+ private void setEntityLoaderFilter(){
+ collectorFilter = filterFactory.entityLoadCollector();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java
new file mode 100644
index 0000000..ba7c802
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Traverses edges in the graph. Either by query or graph traversal. Take an observable of ids, and emits
+ * an observable of ids
+ */
+public interface TraverseFilter extends Filter<Id> {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
new file mode 100644
index 0000000..01ab21f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ElasticSearchQueryExecutor;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+
+/**
+ * A command that will query and load elasticsearch
+ *
+ * On future iteration, this needs to be split into 2 commands 1 that loads the candidate results and validates the
+ * versions then another that will search and load
+ *
+ * TODO, split this into 3 seperate observables
+ *
+ * 1) An observable that emits candidate results 2) An observable that validates versions by uuid (for Traverse
+ * commands) 3) An observbale that emits Results as a collector (for final commands)
+ */
+public abstract class AbstractQueryElasticSearchCollectorFilter extends AbstractFilter<Results, Integer>
+ implements CollectorFilter<Results> {
+
+
+ protected final ApplicationEntityIndex entityIndex;
+ protected final Query query;
+ private int limit;
+
+
+ @Inject
+ protected AbstractQueryElasticSearchCollectorFilter( final ApplicationEntityIndex entityIndex, final Query query ) {
+ this.entityIndex = entityIndex;
+ this.query = query;
+ }
+
+
+ @Override
+ public Observable<Results> call( final Observable<Id> idObservable ) {
+
+ return idObservable.flatMap( id -> {
+
+ //TODO, refactor this logic to use Observables. make this a TraverseFilter and load entities with the entity loader collector
+ final ResultsLoaderFactory resultsLoaderFactory = getResultsLoaderFactory( id );
+ final SearchEdge searchEdge = getSearchEdge( id );
+ final SearchTypes searchTypes = getSearchTypes();
+
+ final Iterable<Results> executor =
+ new ElasticSearchQueryExecutor( resultsLoaderFactory, entityIndex, applicationScope,
+ searchEdge, searchTypes, query.withLimit( limit ) );
+
+ return Observable.from( executor );
+ } );
+ }
+
+
+ /**
+ * Get the search types
+ */
+ protected abstract SearchTypes getSearchTypes();
+
+ /**
+ * Get the search edge
+ */
+ protected abstract SearchEdge getSearchEdge(final Id incomingId);
+
+
+ /**
+ * Get the results loader factor
+ */
+ protected abstract ResultsLoaderFactory getResultsLoaderFactory( final Id incomingId );
+
+
+ @Override
+ protected Class<Integer> getCursorClass() {
+ return Integer.class;
+ }
+
+
+ @Override
+ public void setLimit( final int limit ) {
+ this.limit = limit;
+ }
+
+
+ /**
+ * Get an entiity ref from the Id. TODO refactor this away
+ * @param id
+ * @return
+ */
+ protected EntityRef getRef(final Id id){
+ return new SimpleEntityRef( id.getType(), id.getUuid() );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
new file mode 100644
index 0000000..f0e8139
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionSearchEdge;
+
+
+/**
+ * Command for querying collections
+ */
+public class QueryCollectionElasticSearchCollectorFilter extends AbstractQueryElasticSearchCollectorFilter {
+
+ private final EntityCollectionManager entityCollectionManager;
+ private final ApplicationEntityIndex applicationEntityIndex;
+ private final String collectionName;
+
+
+ @Inject
+ protected QueryCollectionElasticSearchCollectorFilter( final EntityCollectionManager entityCollectionManager,
+ final ApplicationEntityIndex applicationEntityIndex,
+ @Assisted final String collectionName , @Assisted final Query query ) {
+ super( applicationEntityIndex, query );
+ this.entityCollectionManager = entityCollectionManager;
+ this.applicationEntityIndex = applicationEntityIndex;
+ this.collectionName = collectionName;
+ }
+
+
+ @Override
+ protected SearchTypes getSearchTypes() {
+ final SearchTypes types = SearchTypes.fromTypes( collectionName );
+
+ return types;
+ }
+
+
+ @Override
+ protected SearchEdge getSearchEdge( final Id incomingId ) {
+ final SearchEdge searchEdge = createCollectionSearchEdge( incomingId, collectionName );
+
+ return searchEdge;
+ }
+
+
+ @Override
+ protected ResultsLoaderFactory getResultsLoaderFactory( final Id id ) {
+
+ final EntityRef entityRef = getRef( id );
+ return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, applicationEntityIndex, entityRef,
+ collectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
new file mode 100644
index 0000000..2b380df
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchEdge;
+
+
+/**
+ * Command for querying connections
+ */
+public class QueryConnectionElasticSearchCollectorFilter extends AbstractQueryElasticSearchCollectorFilter {
+
+ private final EntityCollectionManager entityCollectionManager;
+ private final ApplicationEntityIndex applicationEntityIndex;
+ private final String connectionName;
+
+
+ @Inject
+ protected QueryConnectionElasticSearchCollectorFilter( final EntityCollectionManager entityCollectionManager,
+ final ApplicationEntityIndex applicationEntityIndex,
+ @Assisted final String connectionName,
+ @Assisted final Query query ) {
+ super( applicationEntityIndex, query );
+ this.entityCollectionManager = entityCollectionManager;
+ this.applicationEntityIndex = applicationEntityIndex;
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected SearchTypes getSearchTypes() {
+
+ final SearchTypes searchTypes = SearchTypes.fromNullableTypes( query.getEntityType() );
+
+ return searchTypes;
+ }
+
+
+ @Override
+ protected SearchEdge getSearchEdge( final Id id ) {
+ final SearchEdge searchEdge = createConnectionSearchEdge( id, connectionName );
+
+ return searchEdge;
+ }
+
+
+ @Override
+ protected ResultsLoaderFactory getResultsLoaderFactory( final Id id ) {
+ final EntityRef entityRef = getRef( id );
+ return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, applicationEntityIndex, entityRef,
+ connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java
new file mode 100644
index 0000000..cc96633
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class CollectionRefsVerifier extends VersionVerifier {
+
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+ List<EntityRef> refs = new ArrayList<EntityRef>(ids.size());
+ for ( Id id : ids ) {
+ refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+ }
+ return Results.fromRefList( refs );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..94c91d9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public class CollectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+ private final EntityCollectionManager entityCollectionManager;
+ private final ApplicationEntityIndex applicationEntityIndex;
+
+
+ public CollectionResultsLoaderFactoryImpl( final EntityCollectionManager entityCollectionManager,
+ final ApplicationEntityIndex applicationEntityIndex ) {
+ this.entityCollectionManager = entityCollectionManager;
+ this.applicationEntityIndex = applicationEntityIndex;
+ }
+
+
+ @Override
+ public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope,
+ final Query.Level resultsLevel ) {
+
+ ResultsVerifier verifier;
+
+ if ( resultsLevel == Query.Level.REFS ) {
+ verifier = new CollectionRefsVerifier();
+ }
+ else if ( resultsLevel == Query.Level.IDS ) {
+ verifier = new IdsVerifier();
+ }
+ else {
+ verifier = new EntityVerifier( Query.MAX_LIMIT );
+ }
+
+ return new FilteringLoader( entityCollectionManager, applicationEntityIndex, verifier, applicationScope,
+ scope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java
new file mode 100644
index 0000000..6b7bdde
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+
+
+/**
+ * Verifier for creating connections
+ */
+public class ConnectionRefsVerifier extends VersionVerifier {
+
+
+ private final EntityRef ownerId;
+ private final String connectionType;
+
+
+ public ConnectionRefsVerifier( final EntityRef ownerId, final String connectionType ) {
+ this.ownerId = ownerId;
+ this.connectionType = connectionType;
+ }
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+ List<ConnectionRef> refs = new ArrayList<>();
+ for ( Id id : ids ) {
+ refs.add( new ConnectionRefImpl( ownerId, connectionType, ref(id.getType(), id.getUuid()) ));
+ }
+
+ return Results.fromConnections( refs );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..55b95a9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public class ConnectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+ private final EntityCollectionManager entityCollectionManager;
+ private final ApplicationEntityIndex applicationEntityIndex;
+ private final EntityRef ownerId;
+ private final String connectionType;
+
+
+ public ConnectionResultsLoaderFactoryImpl( final EntityCollectionManager entityCollectionManager,
+ final ApplicationEntityIndex applicationEntityIndex, final EntityRef ownerId,
+ final String connectionType ) {
+ this.entityCollectionManager = entityCollectionManager;
+ this.applicationEntityIndex = applicationEntityIndex;
+ this.ownerId = ownerId;
+ this.connectionType = connectionType;
+ }
+
+
+ @Override
+ public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope,
+ final Query.Level resultsLevel ) {
+
+ ResultsVerifier verifier;
+
+ if ( resultsLevel == Query.Level.REFS ) {
+ verifier = new ConnectionRefsVerifier( ownerId, connectionType );
+ }
+ else if ( resultsLevel == Query.Level.IDS ) {
+ verifier = new ConnectionRefsVerifier( ownerId, connectionType );
+ ;
+ }
+ else {
+ verifier = new EntityVerifier( Query.MAX_LIMIT );
+ }
+
+ return new FilteringLoader( entityCollectionManager, applicationEntityIndex, verifier, applicationScope, scope );
+ }
+}
[02/20] incubator-usergrid git commit: Changed timeout name to be
clearer
Posted by to...@apache.org.
Changed timeout name to be clearer
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/14c1f36a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/14c1f36a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/14c1f36a
Branch: refs/heads/two-dot-o-dev
Commit: 14c1f36a8730c8b66c7cae1fa43fadfb002cddf3
Parents: 8b97a6d
Author: GERey <gr...@apigee.com>
Authored: Fri Apr 17 14:56:53 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Apr 17 14:56:53 2015 -0700
----------------------------------------------------------------------
.../usergrid/persistence/index/IndexFig.java | 6 +++---
.../persistence/index/impl/EsEntityIndexImpl.java | 18 ++----------------
2 files changed, 5 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/14c1f36a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 2539696..3890ff8 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -104,7 +104,7 @@ public interface IndexFig extends GuicyFig {
* Timeout calls to elasticsearch.
* @return
*/
- public static final String ELASTICSEARCH_TIMEOUT = "elasticsearch.timeout";
+ public static final String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
@Default( "127.0.0.1" )
@@ -218,6 +218,6 @@ public interface IndexFig extends GuicyFig {
long getQueueOfferTimeout();
@Default( "5000" )
- @Key( ELASTICSEARCH_TIMEOUT )
- long getElasticsearchTimeout();
+ @Key( ELASTICSEARCH_QUERY_TIMEOUT )
+ long getElasticsearchQueryTimeout();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/14c1f36a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 9d8d278..f17fd58 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -18,15 +18,12 @@
package org.apache.usergrid.persistence.index.impl;
-import com.codahale.metrics.Meter;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
-import com.google.common.util.concurrent.Futures;
-import com.google.common.util.concurrent.ListenableFuture;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import com.yammer.metrics.core.Clock;
+
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.future.BetterFuture;
@@ -39,7 +36,6 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.map.MapScope;
@@ -49,9 +45,6 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
@@ -61,9 +54,6 @@ import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
-import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
-import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
@@ -75,8 +65,6 @@ import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.*;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
-import org.elasticsearch.indices.InvalidAliasNameException;
-import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -84,8 +72,6 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import rx.Observable;
-import rx.functions.Func1;
import java.io.IOException;
import java.util.ArrayList;
@@ -503,7 +489,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
Timer.Context timeSearchCursor = cursorTimer.time();
- searchResponse = ssrb.execute().actionGet(config.getElasticsearchTimeout());
+ searchResponse = ssrb.execute().actionGet(config.getElasticsearchQueryTimeout());
timeSearchCursor.stop();
}
catch ( Throwable t ) {
[16/20] incubator-usergrid git commit: First pass at new command
layout. Need to test and debug
Posted by to...@apache.org.
First pass at new command layout. Need to test and debug
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0523dc03
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0523dc03
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0523dc03
Branch: refs/heads/two-dot-o-dev
Commit: 0523dc0384bf8a263c06ee37d66f5fbd8977fa0b
Parents: d5978cc
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 18:12:19 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 18:12:19 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 6 +-
.../corepersistence/command/CommandBuilder.java | 111 ---------
.../command/cursor/CursorSerializer.java | 86 -------
.../command/cursor/RequestCursor.java | 45 ----
.../command/cursor/ResponseCursor.java | 56 -----
.../command/read/AbstractCommand.java | 111 ---------
.../corepersistence/command/read/Collector.java | 36 ---
.../corepersistence/command/read/Command.java | 56 -----
.../command/read/ReadCommandFactory.java | 81 -------
.../command/read/TraverseCommand.java | 30 ---
.../AbstractQueryElasticSearchCollector.java | 107 ---------
.../QueryCollectionElasticSearchCollector.java | 62 -----
.../QueryConnectionElasticSearchCollector.java | 62 -----
.../impl/CollectionRefsVerifier.java | 44 ----
.../CollectionResultsLoaderFactoryImpl.java | 59 -----
.../impl/ConnectionRefsVerifier.java | 59 -----
.../ConnectionResultsLoaderFactoryImpl.java | 65 -----
.../impl/ElasticSearchQueryExecutor.java | 238 -------------------
.../read/elasticsearch/impl/EntityVerifier.java | 127 ----------
.../elasticsearch/impl/FilteringLoader.java | 226 ------------------
.../read/elasticsearch/impl/IdsVerifier.java | 46 ----
.../read/elasticsearch/impl/ResultsLoader.java | 43 ----
.../impl/ResultsLoaderFactory.java | 41 ----
.../elasticsearch/impl/ResultsVerifier.java | 52 ----
.../elasticsearch/impl/VersionVerifier.java | 85 -------
.../read/entity/EntityLoadCollector.java | 138 -----------
.../read/graph/AbstractReadGraphCommand.java | 110 ---------
.../read/graph/ReadGraphCollectionCommand.java | 52 ----
.../read/graph/ReadGraphConnectionCommand.java | 52 ----
.../corepersistence/pipeline/DataPipeline.java | 136 +++++++++++
.../pipeline/PipelineBuilderFactory.java | 39 +++
.../pipeline/PipelineModule.java | 40 ++++
.../pipeline/cursor/CursorSerializer.java | 86 +++++++
.../pipeline/cursor/RequestCursor.java | 45 ++++
.../pipeline/cursor/ResponseCursor.java | 56 +++++
.../pipeline/read/AbstractFilter.java | 109 +++++++++
.../pipeline/read/CollectorFilter.java | 36 +++
.../corepersistence/pipeline/read/Filter.java | 57 +++++
.../pipeline/read/FilterFactory.java | 105 ++++++++
.../pipeline/read/ReadPipelineBuilder.java | 102 ++++++++
.../pipeline/read/ReadPipelineBuilderImpl.java | 180 ++++++++++++++
.../pipeline/read/TraverseFilter.java | 30 +++
...stractQueryElasticSearchCollectorFilter.java | 125 ++++++++++
...yCollectionElasticSearchCollectorFilter.java | 83 +++++++
...yConnectionElasticSearchCollectorFilter.java | 84 +++++++
.../impl/CollectionRefsVerifier.java | 44 ++++
.../CollectionResultsLoaderFactoryImpl.java | 65 +++++
.../impl/ConnectionRefsVerifier.java | 59 +++++
.../ConnectionResultsLoaderFactoryImpl.java | 73 ++++++
.../impl/ElasticSearchQueryExecutor.java | 237 ++++++++++++++++++
.../read/elasticsearch/impl/EntityVerifier.java | 127 ++++++++++
.../elasticsearch/impl/FilteringLoader.java | 219 +++++++++++++++++
.../read/elasticsearch/impl/IdsVerifier.java | 46 ++++
.../read/elasticsearch/impl/ResultsLoader.java | 43 ++++
.../impl/ResultsLoaderFactory.java | 41 ++++
.../elasticsearch/impl/ResultsVerifier.java | 52 ++++
.../elasticsearch/impl/VersionVerifier.java | 85 +++++++
.../pipeline/read/entity/EntityIdFilter.java | 59 +++++
.../read/entity/EntityLoadCollectorFilter.java | 140 +++++++++++
.../graph/AbstractReadGraphEdgeByIdFilter.java | 86 +++++++
.../read/graph/AbstractReadGraphFilter.java | 100 ++++++++
.../graph/ReadGraphCollectionByIdFilter.java | 47 ++++
.../read/graph/ReadGraphCollectionFilter.java | 53 +++++
.../graph/ReadGraphConnectionByIdFilter.java | 48 ++++
.../graph/ReadGraphConnectionByTypeFilter.java | 95 ++++++++
.../read/graph/ReadGraphConnectionFilter.java | 53 +++++
.../results/AbstractGraphQueryExecutor.java | 12 +-
.../results/CollectionGraphQueryExecutor.java | 8 +-
.../results/ConnectionGraphQueryExecutor.java | 8 +-
.../graph/impl/SimpleSearchByEdge.java | 4 +-
.../graph/impl/SimpleSearchByIdType.java | 4 +-
.../graph/test/util/EdgeTestUtils.java | 5 +-
72 files changed, 3111 insertions(+), 2301 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index de7fef1..d8f9446 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -32,10 +32,10 @@ import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.results.CollectionGraphQueryExecutor;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.CollectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionGraphQueryExecutor;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ElasticSearchQueryExecutor;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ElasticSearchQueryExecutor;
import org.apache.usergrid.corepersistence.results.QueryExecutor;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
deleted file mode 100644
index d9d6a12..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command;
-
-
-import org.apache.usergrid.corepersistence.command.cursor.RequestCursor;
-import org.apache.usergrid.corepersistence.command.cursor.ResponseCursor;
-import org.apache.usergrid.corepersistence.command.read.Collector;
-import org.apache.usergrid.corepersistence.command.read.Command;
-import org.apache.usergrid.corepersistence.command.read.TraverseCommand;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-
-
-/**
- * A builder that will allow us to build a traversal command for execution
- */
-public class CommandBuilder {
-
-
- private final ApplicationScope applicationScope;
- private final RequestCursor requestCursor;
- private final ResponseCursor responseCursor;
- private final int limit;
-
- private int count = 0;
-
- private Observable<Id> currentObservable;
-
-
- /**
- * Our first pass, where we implement our start point as an Id until we can use this to perform our entire
- * traversal. Eventually as we untangle the existing Query service nightmare, the sourceId will be remove and should
- * only be traversed from the root application
- */
- public CommandBuilder( final ApplicationScope applicationScope, final Id sourceId,
- final Optional<String> requestCursor, final int limit ) {
-
- this.applicationScope = applicationScope;
- this.limit = limit;
-
- //set the request cursor
- this.requestCursor = new RequestCursor( requestCursor );
-
- //set the response cursor
- this.responseCursor = new ResponseCursor();
-
-
- this.currentObservable = Observable.just( sourceId );
- }
-
-
- /**
- * Add a read command that will read Ids and produce Ids. This is an intermediate traversal operations
- */
- public CommandBuilder withTraverseCommand( final TraverseCommand traverseCommand ) {
-
- setState( traverseCommand );
-
- this.currentObservable = currentObservable.compose( traverseCommand );
-
- return this;
- }
-
-
- /**
- * Build the final collection step, and
- */
- public <T> Observable<T> build( final Collector<T> collector ) {
- setState( collector );
-
- collector.setLimit( limit );
-
- return currentObservable.compose( collector );
- }
-
-
- /**
- * Set the id of the state
- * @param command
- */
- private void setState( Command<?> command ) {
- command.setId( count );
- //done for clarity
- count++;
-
- command.setCursorCaches( requestCursor, responseCursor );
- command.setApplicationScope( applicationScope );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java
deleted file mode 100644
index b45e7da..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/CursorSerializer.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.cursor;
-
-
-import java.io.Serializable;
-
-import com.fasterxml.jackson.core.Base64Variant;
-import com.fasterxml.jackson.core.Base64Variants;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.dataformat.smile.SmileFactory;
-
-
-/**
- * A utility to serialize objects to/from cursors
- */
-public class CursorSerializer {
-
-
- private static final SmileFactory SMILE_FACTORY = new SmileFactory();
-
- private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
-
- private static final Base64Variant VARIANT = Base64Variants.MODIFIED_FOR_URL;
-
-
- /**
- * Serialize the serializable object as a cursor
- */
- public static String asCursor( final Serializable cursor ) {
-
- try {
- return MAPPER.writer( VARIANT ).writeValueAsString( cursor );
- }
- catch ( JsonProcessingException e ) {
- throw new CursorParseException( "Unable to serialize cursor", e );
- }
- }
-
-
- /**
- * Deserialize from the cursor
- * @param cursor
- * @return
- * @throws CursorParseException
- */
- public <T extends Serializable> T fromCursor( final String cursor, final Class<T> cursorClass ) throws CursorParseException {
- try {
-
- final JsonParser parser = MAPPER.getFactory().createParser( cursor );
- return MAPPER.reader( VARIANT ).readValue( parser, cursorClass);
- }
- catch ( Exception e ) {
- throw new CursorParseException( "Unable to serialize cursor", e );
- }
- }
-
-
- /**
- * Thrown when we can't parse a cursor
- */
- public static class CursorParseException extends RuntimeException {
- public CursorParseException( final String message, final Throwable cause ) {
- super( message, cause );
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java
deleted file mode 100644
index 60d54ff..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/RequestCursor.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.cursor;
-
-
-import java.io.Serializable;
-
-import com.google.common.base.Optional;
-
-
-/**
- * A cursor that has been passed in with our request. Adds utils for parsing values
- */
-public class RequestCursor {
-
- public RequestCursor(final Optional<String> cursor){
-
- }
-
-
- /**
- * Get the cursor with the specified id
- */
- public <T extends Serializable> T getCursor( final int id, final Class<T> cursorType ) {
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java
deleted file mode 100644
index 02aae34..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/cursor/ResponseCursor.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.cursor;
-
-
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.Map;
-
-
-/**
- * A cursor used in rendering a response
- */
-public class ResponseCursor {
-
-
-
- /**
- * We use a map b/c some indexes might be skipped
- */
- private Map<Integer, ? super Serializable> cursors = new HashMap<>();
-
- /**
- * Set the possible cursor value into the index. DOES NOT parse the cursor. This is intentional for performance
- */
- public <T extends Serializable> void setCursor( final int id, final T cursor ) {
- cursors.put( id, cursor );
- }
-
-
- private void ensureCapacity() {
-
- }
-
-
- public String encodeAsString() {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java
deleted file mode 100644
index 59e1848..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/AbstractCommand.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read;
-
-
-import java.io.Serializable;
-
-import javax.xml.ws.Response;
-
-import org.apache.usergrid.corepersistence.command.cursor.RequestCursor;
-import org.apache.usergrid.corepersistence.command.cursor.ResponseCursor;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import com.google.common.base.Optional;
-
-
-/**
- * Basic functionality for our commands to handle cursor IO
- */
-public abstract class AbstractCommand<T, C extends Serializable> implements Command<T> {
-
- private int id;
- /**
- * The cache of the cursor that was set when the read was started
- */
- private RequestCursor readCache;
-
- /**
- * The current state of the write cache. Gets updated as we traverse the observables
- */
- private ResponseCursor writeCache;
-
-
- /**
- * The applicationScope
- */
- protected ApplicationScope applicationScope;
-
-
- @Override
- public void setId( final int id ) {
- this.id = id;
- }
-
-
- @Override
- public void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache ) {
- this.readCache = readCache;
- this.writeCache = writeCache;
- }
-
-
- @Override
- public void setApplicationScope( final ApplicationScope applicationScope ) {
- this.applicationScope = applicationScope;
- }
-
-
- /**
- * Return the parsed value of the cursor from the last request, if it exists
- */
- protected Optional<C> getCursor() {
- final C cursor = readCache.getCursor( id, getCursorClass() );
-
- return Optional.fromNullable( cursor );
- }
-
-
-
-
-
- /**
- * Set the cursor value into the new cursor write cache
- * @param newValue
- */
- protected void setCursor(final C newValue){
- writeCache.setCursor( id, newValue );
- }
-
-
- /**
- * Generate our state as a cursor
- * @return
- */
- protected String generateCursor(){
- return writeCache.encodeAsString();
- }
-
- /**
- * Return the class to be used when parsing the cursor
- */
- protected abstract Class<C> getCursorClass();
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java
deleted file mode 100644
index 2804db4..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read;
-
-
-/**
- * A command that is used to reduce our stream of results into a final output
- * @param <T>
- */
-public interface Collector<T> extends Command<T>{
-
- /**
- * Set the prefered result size for the command
- * @param limit
- */
- void setLimit( final int limit );
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java
deleted file mode 100644
index ada47d2..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Command.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read;
-
-
-import org.apache.usergrid.corepersistence.command.cursor.RequestCursor;
-import org.apache.usergrid.corepersistence.command.cursor.ResponseCursor;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Interface for a read command. This takes an input of Id, performs some operation, and emits Id for further
- * processing
- */
-public interface Command<T> extends Observable.Transformer<Id, T> {
-
-
- /**
- * Set the id of this command in it's execution environment
- */
- void setId( final int id );
-
- /**
- * Set the cursor cache into the command
- *
- * @param readCache Set the cache that was used in the request
- * @param writeCache Set the cache to be used when writing the results
- */
- void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache );
-
- /**
- * Set the application scope of the command
- * @param applicationScope
- */
- void setApplicationScope(final ApplicationScope applicationScope);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java
deleted file mode 100644
index 07f300a..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read;
-
-
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.QueryCollectionElasticSearchCollector;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.QueryConnectionElasticSearchCollector;
-import org.apache.usergrid.corepersistence.command.read.entity.EntityLoadCollector;
-import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphCollectionCommand;
-import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphConnectionCommand;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-/**
- * A factory for generating read commands
- */
-public interface ReadCommandFactory {
-
-
- /**
- * Generate a new instance of the command with the specified parameters
- * @param applicationScope
- * @param sourceId
- * @param collectionName
- * @return
- */
- ReadGraphCollectionCommand readGraphCollectionCommand(final ApplicationScope applicationScope, final Id sourceId, final String collectionName);
-
- /**
- * Generate a new instance of the command with the specified parameters
- * @param applicationScope
- * @param sourceId
- * @param connectionName
- * @return
- */
- ReadGraphConnectionCommand readGraphConnectionCommand(final ApplicationScope applicationScope, final Id sourceId, final String connectionName);
-
- /**
- * Generate a new instance of the command with the specified parameters
- * @param applicationScope
- * @return
- */
- EntityLoadCollector entityLoadCollector(final ApplicationScope applicationScope);
-
- /**
- * Generate a new instance of the command with the specified parameters
- * @param applicationScope
- * @param sourceId
- * @param collectionName
- * @return
- */
- QueryCollectionElasticSearchCollector queryCollectionElasticSearchCollector(final ApplicationScope applicationScope, final Id sourceId, final String collectionName);
-
-
- /**
- * Generate a new instance of the command with the specified parameters
- * @param applicationScope
- * @param sourceId
- * @param connectionName
- * @return
- */
- QueryConnectionElasticSearchCollector queryConnectionElasticSearchCollector(final ApplicationScope applicationScope, final Id sourceId, final String connectionName);
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java
deleted file mode 100644
index c8fe2f8..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/TraverseCommand.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read;
-
-
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-/**
- * Traverses edges in the graph. Either by query or graph traversal. Take an observable of ids, and emits
- * an observable of ids
- */
-public interface TraverseCommand extends Command<Id> {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java
deleted file mode 100644
index 18cdeb0..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch;
-
-
-import java.util.Iterator;
-
-import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
-import org.apache.usergrid.corepersistence.command.read.Collector;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ElasticSearchQueryExecutor;
-import org.apache.usergrid.corepersistence.results.QueryExecutor;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ResultsLoaderFactory;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.inject.Inject;
-
-import rx.Observable;
-
-
-/**
- * A command that will query and load elasticsearch
- *
- * On future iteration, this needs to be split into 2 commands 1 that loads the candidate results and validates the
- * versions then another that will search and load
- *
- * TODO, split this into 3 seperate observables
- *
- * 1) An observable that emits candidate results
- * 2) An observable that validates versions by uuid (for Traverse commands)
- * 3) An observbale that emits Results as a collector (for final commands)
- */
-public abstract class AbstractQueryElasticSearchCollector extends AbstractCommand<Results, Integer> implements
- Collector<Results> {
-
-
- private final ResultsLoaderFactory resultsLoaderFactory;
- private final ApplicationEntityIndex entityIndex;
- private final ApplicationScope applicationScope;
- private final SearchEdge indexScope;
- private final SearchTypes searchTypes;
- private final Query query;
- private int limit;
-
-
-
- protected AbstractQueryElasticSearchCollector( final ApplicationEntityIndex entityIndex,
- final ApplicationScope applicationScope, final SearchEdge indexScope,
- final SearchTypes searchTypes, final Query query ) {
- this.entityIndex = entityIndex;
- this.applicationScope = applicationScope;
- this.indexScope = indexScope;
- this.searchTypes = searchTypes;
- this.query = query;
- this.resultsLoaderFactory = getResultsLoaderFactory();
- }
-
-
- @Override
- public Observable<Results> call( final Observable<Id> idObservable ) {
- final Iterable<Results> executor =
- new ElasticSearchQueryExecutor( resultsLoaderFactory, entityIndex, applicationScope, indexScope, searchTypes,
- query.withLimit( limit ));
-
- return Observable.from(executor);
- }
-
-
- /**
- * Get the results loader factor
- * @return
- */
- protected abstract ResultsLoaderFactory getResultsLoaderFactory();
-
- @Override
- protected Class<Integer> getCursorClass() {
- return Integer.class;
- }
-
-
- @Override
- public void setLimit( final int limit ) {
- this.limit = limit;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java
deleted file mode 100644
index 2bbcc48..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch;
-
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ResultsLoaderFactory;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
-
-import com.google.inject.Inject;
-
-
-/**
- * Command for querying connections
- */
-public class QueryCollectionElasticSearchCollector extends AbstractQueryElasticSearchCollector {
-
- private final ManagerCache managerCache;
- private final EntityRef headEntity;
- private final String connectionName;
-
- @Inject
- protected QueryCollectionElasticSearchCollector( final ApplicationEntityIndex entityIndex,
- final ApplicationScope applicationScope,
- final SearchEdge indexScope, final SearchTypes searchTypes,
- final Query query, final ManagerCache managerCache,
- final EntityRef headEntity, final String connectionName ) {
- super( entityIndex, applicationScope, indexScope, searchTypes, query );
- this.managerCache = managerCache;
- this.headEntity = headEntity;
- this.connectionName = connectionName;
- }
-
-
- @Override
- protected ResultsLoaderFactory getResultsLoaderFactory() {
- return new ConnectionResultsLoaderFactoryImpl( managerCache, headEntity, connectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java
deleted file mode 100644
index 0e2f221..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch;
-
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
-import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ResultsLoaderFactory;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.SearchTypes;
-
-import com.google.inject.Inject;
-
-
-/**
- * Command for querying connections
- */
-public class QueryConnectionElasticSearchCollector extends AbstractQueryElasticSearchCollector {
-
- private final ManagerCache managerCache;
- private final EntityRef headEntity;
- private final String connectionName;
-
- @Inject
- protected QueryConnectionElasticSearchCollector( final ApplicationEntityIndex entityIndex,
- final ApplicationScope applicationScope,
- final SearchEdge indexScope, final SearchTypes searchTypes,
- final Query query, final ManagerCache managerCache,
- final EntityRef headEntity, final String connectionName ) {
- super( entityIndex, applicationScope, indexScope, searchTypes, query );
- this.managerCache = managerCache;
- this.headEntity = headEntity;
- this.connectionName = connectionName;
- }
-
-
- @Override
- protected ResultsLoaderFactory getResultsLoaderFactory() {
- return new ConnectionResultsLoaderFactoryImpl( managerCache, headEntity, connectionName );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java
deleted file mode 100644
index 4b67ae8..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public class CollectionRefsVerifier extends VersionVerifier {
-
-
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
- List<EntityRef> refs = new ArrayList<EntityRef>(ids.size());
- for ( Id id : ids ) {
- refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
- }
- return Results.fromRefList( refs );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
deleted file mode 100644
index f9dd4e1..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.SearchEdge;
-
-
-/**
- * Factory for creating results
- */
-public class CollectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
-
- private final ManagerCache managerCache;
-
-
- public CollectionResultsLoaderFactoryImpl( final ManagerCache managerCache ) {
- this.managerCache = managerCache;
- }
-
-
- @Override
- public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope, final Query.Level resultsLevel ) {
-
- ResultsVerifier verifier;
-
- if ( resultsLevel == Query.Level.REFS ) {
- verifier = new CollectionRefsVerifier();
- }
- else if ( resultsLevel == Query.Level.IDS ) {
- verifier = new IdsVerifier();
- }
- else {
- verifier = new EntityVerifier(Query.MAX_LIMIT);
- }
-
- return new FilteringLoader( managerCache, verifier, applicationScope, scope );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java
deleted file mode 100644
index bffb53a..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.usergrid.persistence.ConnectionRef;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
-
-
-/**
- * Verifier for creating connections
- */
-public class ConnectionRefsVerifier extends VersionVerifier {
-
-
- private final EntityRef ownerId;
- private final String connectionType;
-
-
- public ConnectionRefsVerifier( final EntityRef ownerId, final String connectionType ) {
- this.ownerId = ownerId;
- this.connectionType = connectionType;
- }
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
- List<ConnectionRef> refs = new ArrayList<>();
- for ( Id id : ids ) {
- refs.add( new ConnectionRefImpl( ownerId, connectionType, ref(id.getType(), id.getUuid()) ));
- }
-
- return Results.fromConnections( refs );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
deleted file mode 100644
index 707e933..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.SearchEdge;
-
-
-/**
- * Factory for creating results
- */
-public class ConnectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
-
- private final ManagerCache managerCache;
- private final EntityRef ownerId;
- private final String connectionType;
-
-
- public ConnectionResultsLoaderFactoryImpl( final ManagerCache managerCache, final EntityRef ownerId,
- final String connectionType ) {
- this.managerCache = managerCache;
- this.ownerId = ownerId;
- this.connectionType = connectionType;
- }
-
-
- @Override
- public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope, final Query.Level resultsLevel ) {
-
- ResultsVerifier verifier;
-
- if ( resultsLevel == Query.Level.REFS ) {
- verifier = new ConnectionRefsVerifier( ownerId, connectionType );
- }
- else if ( resultsLevel == Query.Level.IDS ) {
- verifier = new ConnectionRefsVerifier( ownerId, connectionType );;
- }
- else {
- verifier = new EntityVerifier(Query.MAX_LIMIT);
- }
-
- return new FilteringLoader( managerCache, verifier, applicationScope, scope );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
deleted file mode 100644
index c7e8f56..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
+++ /dev/null
@@ -1,238 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.apache.usergrid.corepersistence.results.QueryExecutor;
-import org.apache.usergrid.persistence.index.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-
-import com.google.common.base.Optional;
-
-
-public class ElasticSearchQueryExecutor implements Iterable<Results>, Iterator<Results> {
-
- private static final Logger logger = LoggerFactory.getLogger( ElasticSearchQueryExecutor.class );
-
- private final ResultsLoaderFactory resultsLoaderFactory;
-
- private final ApplicationScope applicationScope;
-
- private final ApplicationEntityIndex entityIndex;
-
- private final SearchEdge indexScope;
-
- private final SearchTypes types;
-
- private final Query query;
-
-
- private Results currentResults;
-
- private boolean moreToLoad = true;
-
-
- public ElasticSearchQueryExecutor( final ResultsLoaderFactory resultsLoaderFactory, final ApplicationEntityIndex entityIndex,
- final ApplicationScope applicationScope, final SearchEdge indexScope,
- final SearchTypes types, final Query query ) {
- this.resultsLoaderFactory = resultsLoaderFactory;
- this.applicationScope = applicationScope;
- this.entityIndex = entityIndex;
- this.indexScope = indexScope;
- this.types = types;
-
- //we must deep copy the query passed. Otherwise we will modify it's state with cursors. Won't fix, not relevant
- //once we start subscribing to streams.
- this.query = new Query(query);
- }
-
-
- @Override
- public Iterator<Results> iterator() {
- return this;
- }
-
-
- private void loadNextPage() {
- // Because of possible stale entities, which are filtered out by buildResults(),
- // we loop until the we've got enough results to satisfy the query limit.
-
- final int maxQueries = 10; // max re-queries to satisfy query limit
-
- final int originalLimit = query.getLimit();
-
- Results results = null;
- int queryCount = 0;
-
-
- CandidateResults crs = null;
-
- while ( queryCount++ < maxQueries ) {
-
- crs = getCandidateResults( query );
-
-
- logger.debug( "Calling build results with crs {}", crs );
- results = buildResults( indexScope, query, crs );
-
- /**
- * In an edge case where we delete stale entities, we could potentially get less results than expected.
- * This will only occur once during the repair phase.
- * We need to ensure that we short circuit before we overflow the requested limit during a repair.
- */
- if ( crs.isEmpty() || !crs.hasOffset() || results.size() > 0 ) { // no results, no cursor, can't get more
- break;
- }
-
-
- //we didn't load anything, but there was a cursor, this means a read repair occured. We have to short
- //circuit to avoid over returning the result set
-
-
- // need to query for more
- // ask for just what we need to satisfy, don't want to exceed limit
- query.setOffsetFromCursor(results.getCursor());
- query.setLimit( originalLimit - results.size() );
-
- logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
- originalLimit, query.getLimit(), queryCount
- } );
- }
-
- //now set our cursor if we have one for the next iteration
- if ( results.hasCursor() ) {
- query.setOffsetFromCursor(results.getCursor());
- moreToLoad = true;
- }
-
- else {
- moreToLoad = false;
- }
-
-
- //set our select subjects into our query if provided
- if(crs != null){
- query.setSelectSubjects( crs.getGetFieldMappings() );
- }
-
-
- //set our current results and the flag
- this.currentResults = results;
- }
-
-
- /**
- * Get the candidates or load the cursor, whichever we require
- * @param query
- * @return
- */
- private CandidateResults getCandidateResults(final Query query){
- final Optional<Integer> cursor = query.getOffset();
- final String queryToExecute = query.getQl().or("select *");
-
- CandidateResults results = cursor.isPresent()
- ? entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get())
- : entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
-
- return results;
- }
-
-
- /**
- * Build results from a set of candidates, and discard those that represent stale indexes.
- *
- * @param query Query that was executed
- * @param crs Candidates to be considered for results
- */
- private Results buildResults( final SearchEdge indexScope, final Query query, final CandidateResults crs ) {
-
- logger.debug( "buildResults() from {} candidates", crs.size() );
-
- //get an instance of our results loader
- final ResultsLoader resultsLoader =
- this.resultsLoaderFactory.getLoader( applicationScope, indexScope, query.getResultsLevel() );
-
- //load the results
- final Results results = resultsLoader.loadResults(crs);
-
- //signal for post processing
- resultsLoader.postProcess();
-
- //set offset into query
- if(crs.getOffset().isPresent()) {
- query.setOffset(crs.getOffset().get());
- }else{
- query.clearOffset();
- }
- results.setCursorFromOffset( query.getOffset() );
-
- logger.debug( "Returning results size {}", results.size() );
-
- return results;
- }
-
-
- @Override
- public boolean hasNext() {
-
- //we've tried to load and it's empty and we have more to load, load the next page
- if ( currentResults == null ) {
- //there's nothing left to load, nothing to do
- if ( !moreToLoad ) {
- return false;
- }
-
- //load the page
-
- loadNextPage();
- }
-
-
- //see if our current results are not null
- return currentResults != null;
- }
-
-
- @Override
- public Results next() {
- if ( !hasNext() ) {
- throw new NoSuchElementException( "No more results present" );
- }
-
- final Results toReturn = currentResults;
-
- currentResults = null;
-
- return toReturn;
- }
-
- @Override
- public void remove() {
- throw new RuntimeException("Remove not implemented!!");
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java
deleted file mode 100644
index 7be8aa4..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-import com.google.common.base.Optional;
-
-
-/**
- * A loader that verifies versions are correct in cassandra and match elasticsearch
- */
-public class EntityVerifier implements ResultsVerifier {
-
- private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
-
- private EntitySet ids;
-
- private Map<Id, org.apache.usergrid.persistence.model.entity.Entity> entityMapping;
-
-
- public EntityVerifier( final int maxSize ) {
- this.entityMapping = new HashMap<>( maxSize );
- }
-
-
- @Override
- public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
- ids = ecm.load( idsToLoad ).toBlocking().last();
- logger.debug("loadResults() asked for {} ids and got {}", idsToLoad.size(), ids.size());
- }
-
-
- @Override
- public boolean isValid( final CandidateResult candidateResult ) {
- final Id entityId = candidateResult.getId();
-
- final MvccEntity savedEntity = ids.getEntity( entityId );
-
- //version wasn't found deindex
- if ( savedEntity == null ) {
- logger.warn( "Version for Entity {}:{} not found", entityId.getType(), entityId.getUuid() );
- return false;
- }
-
- final UUID candidateVersion = candidateResult.getVersion();
- final UUID savedVersion = savedEntity.getVersion();
-
- if ( UUIDComparator.staticCompare( savedVersion, candidateVersion ) > 0 ) {
- logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
- entityId.getUuid(), entityId.getType(), candidateVersion, savedEntity
- } );
-
- return false;
- }
-
-
- final Optional<org.apache.usergrid.persistence.model.entity.Entity> entity = savedEntity.getEntity();
-
- if ( !entity.isPresent() ) {
- logger.warn( "Entity uuid:{} version v:{} is deleted but indexed, this is a bug ",
- entityId.getUuid(), savedEntity.getEntity() );
- return false;
- }
-
- entityMapping.put( entityId, entity.get() );
-
- return true;
- }
-
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
-
- final List<Entity> ugEntities = new ArrayList<>( ids.size() );
-
- for ( final Id id : ids ) {
- final org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityMapping.get( id );
-
- Entity entity = EntityFactory.newEntity( id.getUuid(), id.getType() );
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
- entity.addProperties( entityMap );
- ugEntities.add( entity );
- }
-
- return Results.fromEntities( ugEntities );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java
deleted file mode 100644
index c0afe92..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java
+++ /dev/null
@@ -1,226 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.UUID;
-
-import javax.annotation.Nullable;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.corepersistence.ManagerCache;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.apache.usergrid.persistence.index.SearchEdge;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.index.CandidateResults;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-import com.google.common.base.Function;
-import com.google.common.collect.Collections2;
-
-
-public class FilteringLoader implements ResultsLoader {
-
- private static final Logger logger = LoggerFactory.getLogger( FilteringLoader.class );
-
- private final ManagerCache managerCache;
- private final ResultsVerifier resultsVerifier;
- private final ApplicationScope applicationScope;
- private final SearchEdge indexScope;
- private final EntityIndexBatch indexBatch;
-
-
- /**
- * Create an instance of a filter loader
- *
- * @param managerCache The manager cache to load
- * @param resultsVerifier The verifier to verify the candidate results
- * @param applicationScope The application scope to perform the load
- * @param indexScope The index scope used in the search
- */
- protected FilteringLoader( final ManagerCache managerCache, final ResultsVerifier resultsVerifier,
- final ApplicationScope applicationScope, final SearchEdge indexScope ) {
-
- this.managerCache = managerCache;
- this.resultsVerifier = resultsVerifier;
- this.applicationScope = applicationScope;
- this.indexScope = indexScope;
-
- final ApplicationEntityIndex index = managerCache.getEntityIndex( applicationScope );
-
- indexBatch = index.createBatch();
- }
-
-
- @Override
- public Results loadResults( final CandidateResults crs ) {
-
-
- if ( crs.size() == 0 ) {
- return new Results();
- }
-
-
- // For each entity, holds the index it appears in our candidates for keeping ordering correct
- final Map<Id, Integer> orderIndex = new HashMap<>( crs.size() );
-
- // Maps the entity ids to our candidates
- final Map<Id, CandidateResult> maxCandidateMapping = new HashMap<>( crs.size() );
-
-
- final Iterator<CandidateResult> iter = crs.iterator();
-
-
- // TODO, in this case we're "optimizing" due to the limitations of collection scope.
- // Perhaps we should change the API to just be an application, then an "owner" scope?
-
- // Go through the candidates and group them by scope for more efficient retrieval.
- // Also remove duplicates before we even make a network call
- for ( int i = 0; iter.hasNext(); i++ ) {
-
- final CandidateResult currentCandidate = iter.next();
-
- final Id entityId = currentCandidate.getId();
-
- //check if we've seen this candidate by id
- final CandidateResult previousMax = maxCandidateMapping.get( entityId );
-
- //its not been seen, save it
- if ( previousMax == null ) {
- maxCandidateMapping.put( entityId, currentCandidate );
- orderIndex.put( entityId, i );
- continue;
- }
-
- //we have seen it, compare them
-
- final UUID previousMaxVersion = previousMax.getVersion();
-
- final UUID currentVersion = currentCandidate.getVersion();
-
-
- final CandidateResult toRemove;
- final CandidateResult toKeep;
-
- //current is newer than previous. Remove previous and keep current
- if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
- toRemove = previousMax;
- toKeep = currentCandidate;
- }
- //previously seen value is newer than current. Remove the current and keep the previously seen value
- else {
- toRemove = currentCandidate;
- toKeep = previousMax;
- }
-
- //this is a newer version, we know we already have a stale entity, add it to be cleaned up
-
-
- //de-index it
- logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
- entityId.getUuid(), entityId.getType(), toRemove.getVersion(), toKeep.getVersion()
- } );
-
- //deindex this document, and remove the previous maxVersion
- //we have to deindex this from our ownerId, since this is what gave us the reference
- indexBatch.deindex( indexScope, toRemove );
-
-
- //TODO, fire the entity repair cleanup task here instead of de-indexing
-
- //replace the value with a more current version
- maxCandidateMapping.put( entityId, toKeep );
- orderIndex.put( entityId, i );
- }
-
-
- //now everything is ordered, and older versions are removed. Batch fetch versions to verify
- // existence and correct versions
-
- final TreeMap<Integer, Id> sortedResults = new TreeMap<>();
-
-
- final Collection<Id> idsToLoad =
- Collections2.transform( maxCandidateMapping.values(), new Function<CandidateResult, Id>() {
- @Nullable
- @Override
- public Id apply( @Nullable final CandidateResult input ) {
- //NOTE this is never null, we won't need to check
- return input.getId();
- }
- } );
-
-
- //now using the scope, load the collection
-
- // Get the collection scope and batch load all the versions. We put all entities in
- // app/app for easy retrieval/ unless persistence changes, we never want to read from
- // any scope other than the app, app, scope name scope
- // final CollectionScope collScope = new CollectionScopeImpl(
- // applicationScope.getApplication(), applicationScope.getApplication(), scopeName);
-
- final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( applicationScope );
-
-
- //load the results into the loader for this scope for validation
- resultsVerifier.loadResults( idsToLoad, ecm );
-
- //now let the loader validate each candidate. For instance, the "max" in this candidate
- //could still be a stale result, so it needs validated
- for ( final Id requestedId : idsToLoad ) {
-
- final CandidateResult cr = maxCandidateMapping.get( requestedId );
-
- //ask the loader if this is valid, if not discard it and de-index it
- if ( !resultsVerifier.isValid( cr ) ) {
- indexBatch.deindex( indexScope, cr );
- continue;
- }
-
- //if we get here we're good, we need to add this to our results
- final int candidateIndex = orderIndex.get( requestedId );
-
- sortedResults.put( candidateIndex, requestedId );
- }
-
-
- // NOTE DO NOT execute the batch here.
- // It changes the results and we need consistent paging until we aggregate all results
- return resultsVerifier.getResults( sortedResults.values() );
- }
-
-
- @Override
- public void postProcess() {
- this.indexBatch.execute();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java
deleted file mode 100644
index 4d2bd55..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public class IdsVerifier extends VersionVerifier {
-
- @Override
- public Results getResults( final Collection<Id> ids ) {
-
- final List<UUID> returnIds = new ArrayList<>( ids.size() );
-
- for ( final Id id : ids ) {
- returnIds.add( id.getUuid() );
- }
-
-
- return Results.fromIdList( returnIds );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java
deleted file mode 100644
index fa0e71f..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.index.CandidateResults;
-
-
-/**
- * Interface for loading results
- */
-public interface ResultsLoader {
-
- /**
- * Using the candidate results, load our results. Should filter stale results
- * @param crs The candidate result set
- * @return Results. Null safe, but may be empty
- */
- public Results loadResults( final CandidateResults crs);
-
- /**
- * Post process the load operation
- */
- public void postProcess();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java
deleted file mode 100644
index 14db80e..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.SearchEdge;
-
-
-/**
- * Factory for creating results
- */
-public interface ResultsLoaderFactory {
-
- /**
- * Get the loader for results
- * @param applicationScope The application scope used to load results
- * @param indexScope The index scope used in the search
- * @param
- */
- ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge indexScope,
- final Query.Level resultsLevel );
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java
deleted file mode 100644
index 68515fa..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.Collection;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public interface ResultsVerifier {
-
- /**
- * Load all the candidate ides for verification
- * @param ids The Id's to load
- * @param ecm The entity collection manager
- */
- public void loadResults(Collection<Id> ids, EntityCollectionManager ecm);
-
- /**
- * Return true if the candidate result is a valid result that should be retained. If it should
- * not it should also be removed from the list of possible return values in this loader
- * @param candidateResult
- */
- public boolean isValid(CandidateResult candidateResult);
-
-
- /**
- * Load the result set with the given ids
- * @return
- */
- public Results getResults(Collection<Id> ids);
-}
[03/20] incubator-usergrid git commit: Refactored timeouts for
clarity. Added search timeout I missed. Added write timeout to
EsIndexBufferConsumer for sending requests.
Posted by to...@apache.org.
Refactored timeouts for clarity.
Added search timeout I missed.
Added write timeout to EsIndexBufferConsumer for sending requests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/74f66848
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/74f66848
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/74f66848
Branch: refs/heads/two-dot-o-dev
Commit: 74f66848484bfa372aef9e343ebb6445d45e72ae
Parents: 14c1f36
Author: GERey <gr...@apigee.com>
Authored: Fri Apr 17 15:10:05 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Fri Apr 17 15:10:05 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/usergrid/persistence/index/IndexFig.java | 8 +++++++-
.../usergrid/persistence/index/impl/EsEntityIndexImpl.java | 4 ++--
.../persistence/index/impl/EsIndexBufferConsumerImpl.java | 2 +-
3 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/74f66848/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 3890ff8..fcab21f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -106,6 +106,8 @@ public interface IndexFig extends GuicyFig {
*/
public static final String ELASTICSEARCH_QUERY_TIMEOUT = "elasticsearch.query.timeout";
+ public static final String ELASTICSEARCH_WRITE_TIMEOUT= "elasticsearch.write.timeout";
+
@Default( "127.0.0.1" )
@Key( ELASTICSEARCH_HOSTS )
@@ -219,5 +221,9 @@ public interface IndexFig extends GuicyFig {
@Default( "5000" )
@Key( ELASTICSEARCH_QUERY_TIMEOUT )
- long getElasticsearchQueryTimeout();
+ long getQueryTimeout();
+
+ @Default( "5000" )
+ @Key( ELASTICSEARCH_WRITE_TIMEOUT )
+ long getWriteTimeout();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/74f66848/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f17fd58..9add426 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -452,7 +452,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
Timer.Context timeSearch = searchTimer.time();
- searchResponse = srb.execute().actionGet();
+ searchResponse = srb.execute().actionGet(config.getQueryTimeout());
timeSearch.stop();
}
catch ( Throwable t ) {
@@ -489,7 +489,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
Timer.Context timeSearchCursor = cursorTimer.time();
- searchResponse = ssrb.execute().actionGet(config.getElasticsearchQueryTimeout());
+ searchResponse = ssrb.execute().actionGet(config.getQueryTimeout());
timeSearchCursor.stop();
}
catch ( Throwable t ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/74f66848/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index d064b97..f0f6427 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -315,7 +315,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
final BulkResponse responses;
try {
- responses = bulkRequest.execute().actionGet();
+ responses = bulkRequest.execute().actionGet(config.getWriteTimeout());
} catch (Throwable t) {
log.error("Unable to communicate with elasticsearch");
failureMonitor.fail("Unable to execute batch", t);
[14/20] incubator-usergrid git commit: First pass at new command
layout. Need to test and debug
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
new file mode 100644
index 0000000..ccc5198
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.usergrid.persistence.index.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+public class ElasticSearchQueryExecutor implements Iterable<Results>, Iterator<Results> {
+
+ private static final Logger logger = LoggerFactory.getLogger( ElasticSearchQueryExecutor.class );
+
+ private final ResultsLoaderFactory resultsLoaderFactory;
+
+ private final ApplicationScope applicationScope;
+
+ private final ApplicationEntityIndex entityIndex;
+
+ private final SearchEdge indexScope;
+
+ private final SearchTypes types;
+
+ private final Query query;
+
+
+ private Results currentResults;
+
+ private boolean moreToLoad = true;
+
+
+ public ElasticSearchQueryExecutor( final ResultsLoaderFactory resultsLoaderFactory, final ApplicationEntityIndex entityIndex,
+ final ApplicationScope applicationScope, final SearchEdge indexScope,
+ final SearchTypes types, final Query query ) {
+ this.resultsLoaderFactory = resultsLoaderFactory;
+ this.applicationScope = applicationScope;
+ this.entityIndex = entityIndex;
+ this.indexScope = indexScope;
+ this.types = types;
+
+ //we must deep copy the query passed. Otherwise we will modify it's state with cursors. Won't fix, not relevant
+ //once we start subscribing to streams.
+ this.query = new Query(query);
+ }
+
+
+ @Override
+ public Iterator<Results> iterator() {
+ return this;
+ }
+
+
+ private void loadNextPage() {
+ // Because of possible stale entities, which are filtered out by buildResults(),
+ // we loop until the we've got enough results to satisfy the query limit.
+
+ final int maxQueries = 10; // max re-queries to satisfy query limit
+
+ final int originalLimit = query.getLimit();
+
+ Results results = null;
+ int queryCount = 0;
+
+
+ CandidateResults crs = null;
+
+ while ( queryCount++ < maxQueries ) {
+
+ crs = getCandidateResults( query );
+
+
+ logger.debug( "Calling build results with crs {}", crs );
+ results = buildResults( indexScope, query, crs );
+
+ /**
+ * In an edge case where we delete stale entities, we could potentially get less results than expected.
+ * This will only occur once during the repair phase.
+ * We need to ensure that we short circuit before we overflow the requested limit during a repair.
+ */
+ if ( crs.isEmpty() || !crs.hasOffset() || results.size() > 0 ) { // no results, no cursor, can't get more
+ break;
+ }
+
+
+ //we didn't load anything, but there was a cursor, this means a read repair occured. We have to short
+ //circuit to avoid over returning the result set
+
+
+ // need to query for more
+ // ask for just what we need to satisfy, don't want to exceed limit
+ query.setOffsetFromCursor(results.getCursor());
+ query.setLimit( originalLimit - results.size() );
+
+ logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
+ originalLimit, query.getLimit(), queryCount
+ } );
+ }
+
+ //now set our cursor if we have one for the next iteration
+ if ( results.hasCursor() ) {
+ query.setOffsetFromCursor(results.getCursor());
+ moreToLoad = true;
+ }
+
+ else {
+ moreToLoad = false;
+ }
+
+
+ //set our select subjects into our query if provided
+ if(crs != null){
+ query.setSelectSubjects( crs.getGetFieldMappings() );
+ }
+
+
+ //set our current results and the flag
+ this.currentResults = results;
+ }
+
+
+ /**
+ * Get the candidates or load the cursor, whichever we require
+ * @param query
+ * @return
+ */
+ private CandidateResults getCandidateResults(final Query query){
+ final Optional<Integer> cursor = query.getOffset();
+ final String queryToExecute = query.getQl().or("select *");
+
+ CandidateResults results = cursor.isPresent()
+ ? entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get())
+ : entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
+
+ return results;
+ }
+
+
+ /**
+ * Build results from a set of candidates, and discard those that represent stale indexes.
+ *
+ * @param query Query that was executed
+ * @param crs Candidates to be considered for results
+ */
+ private Results buildResults( final SearchEdge indexScope, final Query query, final CandidateResults crs ) {
+
+ logger.debug( "buildResults() from {} candidates", crs.size() );
+
+ //get an instance of our results loader
+ final ResultsLoader resultsLoader =
+ this.resultsLoaderFactory.getLoader( applicationScope, indexScope, query.getResultsLevel() );
+
+ //load the results
+ final Results results = resultsLoader.loadResults(crs);
+
+ //signal for post processing
+ resultsLoader.postProcess();
+
+ //set offset into query
+ if(crs.getOffset().isPresent()) {
+ query.setOffset(crs.getOffset().get());
+ }else{
+ query.clearOffset();
+ }
+ results.setCursorFromOffset( query.getOffset() );
+
+ logger.debug( "Returning results size {}", results.size() );
+
+ return results;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+
+ //we've tried to load and it's empty and we have more to load, load the next page
+ if ( currentResults == null ) {
+ //there's nothing left to load, nothing to do
+ if ( !moreToLoad ) {
+ return false;
+ }
+
+ //load the page
+
+ loadNextPage();
+ }
+
+
+ //see if our current results are not null
+ return currentResults != null;
+ }
+
+
+ @Override
+ public Results next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more results present" );
+ }
+
+ final Results toReturn = currentResults;
+
+ currentResults = null;
+
+ return toReturn;
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Remove not implemented!!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/EntityVerifier.java
new file mode 100644
index 0000000..d73c731
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/EntityVerifier.java
@@ -0,0 +1,127 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Optional;
+
+
+/**
+ * A loader that verifies versions are correct in cassandra and match elasticsearch
+ */
+public class EntityVerifier implements ResultsVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
+
+ private EntitySet ids;
+
+ private Map<Id, org.apache.usergrid.persistence.model.entity.Entity> entityMapping;
+
+
+ public EntityVerifier( final int maxSize ) {
+ this.entityMapping = new HashMap<>( maxSize );
+ }
+
+
+ @Override
+ public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
+ ids = ecm.load( idsToLoad ).toBlocking().last();
+ logger.debug("loadResults() asked for {} ids and got {}", idsToLoad.size(), ids.size());
+ }
+
+
+ @Override
+ public boolean isValid( final CandidateResult candidateResult ) {
+ final Id entityId = candidateResult.getId();
+
+ final MvccEntity savedEntity = ids.getEntity( entityId );
+
+ //version wasn't found deindex
+ if ( savedEntity == null ) {
+ logger.warn( "Version for Entity {}:{} not found", entityId.getType(), entityId.getUuid() );
+ return false;
+ }
+
+ final UUID candidateVersion = candidateResult.getVersion();
+ final UUID savedVersion = savedEntity.getVersion();
+
+ if ( UUIDComparator.staticCompare( savedVersion, candidateVersion ) > 0 ) {
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), candidateVersion, savedEntity
+ } );
+
+ return false;
+ }
+
+
+ final Optional<org.apache.usergrid.persistence.model.entity.Entity> entity = savedEntity.getEntity();
+
+ if ( !entity.isPresent() ) {
+ logger.warn( "Entity uuid:{} version v:{} is deleted but indexed, this is a bug ",
+ entityId.getUuid(), savedEntity.getEntity() );
+ return false;
+ }
+
+ entityMapping.put( entityId, entity.get() );
+
+ return true;
+ }
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+
+ final List<Entity> ugEntities = new ArrayList<>( ids.size() );
+
+ for ( final Id id : ids ) {
+ final org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityMapping.get( id );
+
+ Entity entity = EntityFactory.newEntity( id.getUuid(), id.getType() );
+
+ Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
+ entity.addProperties( entityMap );
+ ugEntities.add( entity );
+ }
+
+ return Results.fromEntities( ugEntities );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/FilteringLoader.java
new file mode 100644
index 0000000..ade64a2
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/FilteringLoader.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.index.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+
+
+public class FilteringLoader implements ResultsLoader {
+
+ private static final Logger logger = LoggerFactory.getLogger( FilteringLoader.class );
+
+ private final EntityCollectionManager entityCollectionManager;
+ private final ResultsVerifier resultsVerifier;
+ private final ApplicationScope applicationScope;
+ private final SearchEdge indexScope;
+ private final EntityIndexBatch indexBatch;
+
+
+ /**
+ * Create an instance of a filter loader
+ *
+ * @param entityCollectionManager The entityCollectionManagerFactory
+ * @param resultsVerifier The verifier to verify the candidate results
+ * @param applicationScope The application scope to perform the load
+ * @param indexScope The index scope used in the search
+ */
+ protected FilteringLoader( final EntityCollectionManager entityCollectionManager, final ApplicationEntityIndex applicationEntityIndex, final ResultsVerifier resultsVerifier,
+ final ApplicationScope applicationScope, final SearchEdge indexScope ) {
+
+ this.entityCollectionManager = entityCollectionManager;
+ this.resultsVerifier = resultsVerifier;
+ this.applicationScope = applicationScope;
+ this.indexScope = indexScope;
+
+ indexBatch = applicationEntityIndex.createBatch();
+ }
+
+
+ @Override
+ public Results loadResults( final CandidateResults crs ) {
+
+
+ if ( crs.size() == 0 ) {
+ return new Results();
+ }
+
+
+ // For each entity, holds the index it appears in our candidates for keeping ordering correct
+ final Map<Id, Integer> orderIndex = new HashMap<>( crs.size() );
+
+ // Maps the entity ids to our candidates
+ final Map<Id, CandidateResult> maxCandidateMapping = new HashMap<>( crs.size() );
+
+
+ final Iterator<CandidateResult> iter = crs.iterator();
+
+
+ // TODO, in this case we're "optimizing" due to the limitations of collection scope.
+ // Perhaps we should change the API to just be an application, then an "owner" scope?
+
+ // Go through the candidates and group them by scope for more efficient retrieval.
+ // Also remove duplicates before we even make a network call
+ for ( int i = 0; iter.hasNext(); i++ ) {
+
+ final CandidateResult currentCandidate = iter.next();
+
+ final Id entityId = currentCandidate.getId();
+
+ //check if we've seen this candidate by id
+ final CandidateResult previousMax = maxCandidateMapping.get( entityId );
+
+ //its not been seen, save it
+ if ( previousMax == null ) {
+ maxCandidateMapping.put( entityId, currentCandidate );
+ orderIndex.put( entityId, i );
+ continue;
+ }
+
+ //we have seen it, compare them
+
+ final UUID previousMaxVersion = previousMax.getVersion();
+
+ final UUID currentVersion = currentCandidate.getVersion();
+
+
+ final CandidateResult toRemove;
+ final CandidateResult toKeep;
+
+ //current is newer than previous. Remove previous and keep current
+ if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
+ toRemove = previousMax;
+ toKeep = currentCandidate;
+ }
+ //previously seen value is newer than current. Remove the current and keep the previously seen value
+ else {
+ toRemove = currentCandidate;
+ toKeep = previousMax;
+ }
+
+ //this is a newer version, we know we already have a stale entity, add it to be cleaned up
+
+
+ //de-index it
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), toRemove.getVersion(), toKeep.getVersion()
+ } );
+
+ //deindex this document, and remove the previous maxVersion
+ //we have to deindex this from our ownerId, since this is what gave us the reference
+ indexBatch.deindex( indexScope, toRemove );
+
+
+ //TODO, fire the entity repair cleanup task here instead of de-indexing
+
+ //replace the value with a more current version
+ maxCandidateMapping.put( entityId, toKeep );
+ orderIndex.put( entityId, i );
+ }
+
+
+ //now everything is ordered, and older versions are removed. Batch fetch versions to verify
+ // existence and correct versions
+
+ final TreeMap<Integer, Id> sortedResults = new TreeMap<>();
+
+
+ final Collection<Id> idsToLoad =
+ Collections2.transform( maxCandidateMapping.values(), new Function<CandidateResult, Id>() {
+ @Nullable
+ @Override
+ public Id apply( @Nullable final CandidateResult input ) {
+ //NOTE this is never null, we won't need to check
+ return input.getId();
+ }
+ } );
+
+
+ //now using the scope, load the collection
+
+
+
+ //load the results into the loader for this scope for validation
+ resultsVerifier.loadResults( idsToLoad, entityCollectionManager );
+
+ //now let the loader validate each candidate. For instance, the "max" in this candidate
+ //could still be a stale result, so it needs validated
+ for ( final Id requestedId : idsToLoad ) {
+
+ final CandidateResult cr = maxCandidateMapping.get( requestedId );
+
+ //ask the loader if this is valid, if not discard it and de-index it
+ if ( !resultsVerifier.isValid( cr ) ) {
+ indexBatch.deindex( indexScope, cr );
+ continue;
+ }
+
+ //if we get here we're good, we need to add this to our results
+ final int candidateIndex = orderIndex.get( requestedId );
+
+ sortedResults.put( candidateIndex, requestedId );
+ }
+
+
+ // NOTE DO NOT execute the batch here.
+ // It changes the results and we need consistent paging until we aggregate all results
+ return resultsVerifier.getResults( sortedResults.values() );
+ }
+
+
+ @Override
+ public void postProcess() {
+ this.indexBatch.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/IdsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/IdsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/IdsVerifier.java
new file mode 100644
index 0000000..4a3bfcd
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/IdsVerifier.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class IdsVerifier extends VersionVerifier {
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+
+ final List<UUID> returnIds = new ArrayList<>( ids.size() );
+
+ for ( final Id id : ids ) {
+ returnIds.add( id.getUuid() );
+ }
+
+
+ return Results.fromIdList( returnIds );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoader.java
new file mode 100644
index 0000000..c2a3e9a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoader.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.CandidateResults;
+
+
+/**
+ * Interface for loading results
+ */
+public interface ResultsLoader {
+
+ /**
+ * Using the candidate results, load our results. Should filter stale results
+ * @param crs The candidate result set
+ * @return Results. Null safe, but may be empty
+ */
+ public Results loadResults( final CandidateResults crs);
+
+ /**
+ * Post process the load operation
+ */
+ public void postProcess();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoaderFactory.java
new file mode 100644
index 0000000..3ccca1b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsLoaderFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public interface ResultsLoaderFactory {
+
+ /**
+ * Get the loader for results
+ * @param applicationScope The application scope used to load results
+ * @param indexScope The index scope used in the search
+ * @param
+ */
+ ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge indexScope,
+ final Query.Level resultsLevel );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsVerifier.java
new file mode 100644
index 0000000..fe72ca2
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ResultsVerifier.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.Collection;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public interface ResultsVerifier {
+
+ /**
+ * Load all the candidate ides for verification
+ * @param ids The Id's to load
+ * @param ecm The entity collection manager
+ */
+ public void loadResults(Collection<Id> ids, EntityCollectionManager ecm);
+
+ /**
+ * Return true if the candidate result is a valid result that should be retained. If it should
+ * not it should also be removed from the list of possible return values in this loader
+ * @param candidateResult
+ */
+ public boolean isValid(CandidateResult candidateResult);
+
+
+ /**
+ * Load the result set with the given ids
+ * @return
+ */
+ public Results getResults(Collection<Id> ids);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/VersionVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/VersionVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/VersionVerifier.java
new file mode 100644
index 0000000..c49fb28
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/VersionVerifier.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.Collection;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+
+
+/**
+ * A loader that verifies versions are correct in Cassandra and match ElasticSearch
+ */
+public abstract class VersionVerifier implements ResultsVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( VersionVerifier.class );
+
+ private VersionSet ids;
+
+
+ @Override
+ public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
+ ids = ecm.getLatestVersion( idsToLoad ).toBlocking().last();
+ }
+
+
+ @Override
+ public boolean isValid( final CandidateResult candidateResult ) {
+ final Id entityId = candidateResult.getId();
+
+ final MvccLogEntry version = ids.getMaxVersion( entityId );
+
+ //version wasn't found ,deindex
+ if ( version == null ) {
+ logger.warn( "Version for Entity {}:{} not found",
+ entityId.getUuid(), entityId.getUuid() );
+
+ return false;
+ }
+
+ final UUID savedVersion = version.getVersion();
+
+ if ( UUIDComparator.staticCompare( savedVersion, candidateResult.getVersion() ) > 0 ) {
+ logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] {
+ entityId.getUuid(),
+ entityId.getType(),
+ candidateResult.getVersion(),
+ savedVersion
+ } );
+
+ return false;
+ }
+
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityIdFilter.java
new file mode 100644
index 0000000..5734a5b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityIdFilter.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.entity;
+
+
+import java.io.Serializable;
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.TraverseFilter;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+
+/**
+ * This command is a stopgap to make migrating 1.0 code easier. Once full traversal has been implemented, this should
+ * be removed
+ */
+public class EntityIdFilter extends AbstractFilter<Id, Serializable> implements TraverseFilter {
+
+ private final Id entityId;
+
+
+ @Inject
+ public EntityIdFilter( @Assisted final Id entityId ) {this.entityId = entityId;}
+
+
+ @Override
+ protected Class<Serializable> getCursorClass() {
+ //no op
+ return null;
+ }
+
+
+ @Override
+ public Observable<Id> call( final Observable<Id> idObservable ) {
+ return Observable.just( entityId );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java
new file mode 100644
index 0000000..74f626d
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/entity/EntityLoadCollectorFilter.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.entity;
+
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFilter;
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+
+/**
+ * Loads entities from a set of Ids.
+ *
+ * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
+ */
+public class EntityLoadCollectorFilter extends AbstractFilter<Results, Serializable>
+ implements CollectorFilter<Results> {
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private final ApplicationScope applicationScope;
+ private int resultSize;
+
+
+ @Inject
+ public EntityLoadCollectorFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final ApplicationScope applicationScope ) {
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.applicationScope = applicationScope;
+ }
+
+
+ @Override
+ protected Class<Serializable> getCursorClass() {
+ return null;
+ }
+
+
+ @Override
+ public Observable<Results> call( final Observable<Id> observable ) {
+
+
+ /**
+ * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
+ * objects
+ */
+
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ final Observable<EntitySet> entitySetObservable = observable.buffer( resultSize ).flatMap(
+ bufferedIds -> Observable.just( bufferedIds ).flatMap( ids -> entityCollectionManager.load( ids ) ) );
+
+
+ final Observable<Results> resultsObservable = entitySetObservable
+
+ .flatMap( entitySet -> {
+
+ //get our entites and filter missing ones, then collect them into a results object
+ final Observable<MvccEntity> mvccEntityObservable = Observable.from( entitySet.getEntities() );
+
+ //convert them to our old entity model, then filter nulls, meaning they weren't found
+ return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter( entity -> entity == null )
+
+ //convert them to a list, then map them into results
+ .toList().map( entities -> {
+ final Results results = Results.fromEntities( entities );
+ results.setCursor( generateCursor() );
+
+ return results;
+ } )
+ //if no results are present, return an empty results
+ .singleOrDefault( new Results( ) );
+ } );
+
+
+ return resultsObservable;
+ }
+
+ /**
+ * Map a new cp entity to an old entity. May be null if not present
+ */
+
+
+ private org.apache.usergrid.persistence.Entity mapEntity( final MvccEntity mvccEntity ) {
+ if ( !mvccEntity.getEntity().isPresent() ) {
+ return null;
+ }
+
+
+ final Entity cpEntity = mvccEntity.getEntity().get();
+ final Id entityId = cpEntity.getId();
+
+ org.apache.usergrid.persistence.Entity entity =
+ EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
+
+ Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
+ entity.addProperties( entityMap );
+
+ return entity;
+ }
+
+
+ @Override
+ public void setLimit( final int limit ) {
+ this.resultSize = limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java
new file mode 100644
index 0000000..2e8f041
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphEdgeByIdFilter.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.graph;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.TraverseFilter;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+
+/**
+ * Filter should take and Id and a graph edge, and ensure the connection between the two exists
+ */
+public abstract class AbstractReadGraphEdgeByIdFilter extends AbstractFilter<Id, Id> implements TraverseFilter {
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final Id targetId;
+
+
+ @Inject
+ public AbstractReadGraphEdgeByIdFilter( final GraphManagerFactory graphManagerFactory, @Assisted final Id
+ targetId ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.targetId = targetId;
+ }
+
+
+ @Override
+ protected Class<Id> getCursorClass() {
+ //no op
+ return null;
+ }
+
+
+ @Override
+ public Observable<Id> call( final Observable<Id> idObservable ) {
+
+ final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+
+ return idObservable.flatMap( id -> {
+ final String edgeTypeName = getEdgeName();
+
+ //create our search
+ final SearchByEdge searchByEdge =
+ new SimpleSearchByEdge( id, edgeTypeName, targetId, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ Optional.absent() );
+
+ //load the versions of the edge, take the first since that's all we need to validate existence, then emit the target node
+ return gm.loadEdgeVersions( searchByEdge ).take( 1 ).map( edge -> edge.getTargetNode() );
+ } );
+ }
+
+
+ /**
+ * Get the name of the edge to be used in the seek
+ */
+ protected abstract String getEdgeName();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
new file mode 100644
index 0000000..4bdcdad
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/AbstractReadGraphFilter.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.graph;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.TraverseFilter;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Func1;
+
+
+/**
+ * Command for reading graph edges
+ */
+public abstract class AbstractReadGraphFilter extends AbstractFilter<Id, Edge>
+ implements TraverseFilter {
+
+ private final GraphManagerFactory graphManagerFactory;
+
+
+ /**
+ * Create a new instance of our command
+ * @param graphManagerFactory
+ */
+ public AbstractReadGraphFilter( final GraphManagerFactory graphManagerFactory ) {
+ this.graphManagerFactory = graphManagerFactory;
+ }
+
+
+ @Override
+ public Observable<Id> call( final Observable<Id> observable ) {
+
+ //get the graph manager
+ final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //set our our constant state
+ final Optional<Edge> startFromCursor = getCursor();
+
+ final String edgeName = getEdgeTypeName();
+
+
+ //return all ids that are emitted from this edge
+ return observable.flatMap( new Func1<Id, Observable<Id>>() {
+
+ @Override
+ public Observable<Id> call( final Id id ) {
+
+ final SimpleSearchByEdgeType search = new SimpleSearchByEdgeType(id,edgeName, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, startFromCursor );
+
+ /**
+ * TODO, pass a message with pointers to our cursor values to be generated later
+ */
+ return graphManager.loadEdgesFromSource( search ).doOnNext( edge -> setCursor( edge ) ).map(
+ edge -> edge.getTargetNode() );
+ }
+ } );
+ }
+
+
+ @Override
+ protected Class<Edge> getCursorClass() {
+ return Edge.class;
+ }
+
+
+
+ /**
+ * Get the edge type name we should use when traversing
+ * @return
+ */
+ protected abstract String getEdgeTypeName();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
new file mode 100644
index 0000000..2ccad67
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionByIdFilter.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.graph;
+
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.assistedinject.Assisted;
+
+
+/**
+ * Read an edge in the graph to verify it's existence by id
+ */
+public class ReadGraphCollectionByIdFilter extends AbstractReadGraphEdgeByIdFilter{
+
+ private final String collectionName;
+
+ public ReadGraphCollectionByIdFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName, @Assisted final Id targetId ) {
+ super( graphManagerFactory, targetId );
+ this.collectionName = collectionName;
+ }
+
+
+ @Override
+ protected String getEdgeName() {
+ return CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
new file mode 100644
index 0000000..3f0a70a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphCollectionFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.graph;
+
+
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
+
+
+/**
+ * Command for reading graph edges on a collection
+ */
+public class ReadGraphCollectionFilter extends AbstractReadGraphFilter {
+
+ private final String collectionName;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ @Inject
+ public ReadGraphCollectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String collectionName ) {
+ super( graphManagerFactory );
+ this.collectionName = collectionName;
+ }
+
+
+ @Override
+ protected String getEdgeTypeName() {
+ return getCollectionScopeNameFromCollectionName( collectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
new file mode 100644
index 0000000..f16867d
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByIdFilter.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.graph;
+
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.assistedinject.Assisted;
+
+
+/**
+ * Read an edge in the graph to verify it's existence by id
+ */
+public class ReadGraphConnectionByIdFilter extends AbstractReadGraphEdgeByIdFilter{
+
+ private final String connectionName;
+
+ public ReadGraphConnectionByIdFilter( final GraphManagerFactory graphManagerFactory,
+ @Assisted final String connectionName, @Assisted final Id targetId ) {
+ super( graphManagerFactory, targetId );
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected String getEdgeName() {
+ return CpNamingUtils.getEdgeTypeFromConnectionType( connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
new file mode 100644
index 0000000..3057111
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionByTypeFilter.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.graph;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.TraverseFilter;
+import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Func1;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
+
+
+/**
+ * Command for reading graph edges on a connection
+ */
+public class ReadGraphConnectionByTypeFilter extends AbstractFilter<Id, Edge> implements TraverseFilter {
+
+ private final GraphManagerFactory graphManagerFactory;
+ private final String connectionName;
+ private final String entityType;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ public ReadGraphConnectionByTypeFilter( final GraphManagerFactory graphManagerFactory,
+ @Assisted final String connectionName, @Assisted final String entityType ) {
+ this.graphManagerFactory = graphManagerFactory;
+ this.connectionName = connectionName;
+ this.entityType = entityType;
+ }
+
+
+ @Override
+ public Observable<Id> call( final Observable<Id> observable ) {
+
+ //get the graph manager
+ final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
+
+ //set our our constant state
+ final Optional<Edge> startFromCursor = getCursor();
+
+ final String edgeName = getConnectionScopeName( connectionName );
+
+
+ //return all ids that are emitted from this edge
+ return observable.flatMap( id -> {
+
+ final SimpleSearchByIdType search =
+ new SimpleSearchByIdType( id, edgeName, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ entityType, startFromCursor );
+
+ /**
+ * TODO, pass a message with pointers to our cursor values to be generated later
+ */
+ return graphManager.loadEdgesFromSourceByType( search ).doOnNext( edge -> setCursor( edge ) ).map(
+ edge -> edge.getTargetNode() );
+ } );
+ }
+
+
+ @Override
+ protected Class<Edge> getCursorClass() {
+ return Edge.class;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
new file mode 100644
index 0000000..49360f6
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/graph/ReadGraphConnectionFilter.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.graph;
+
+
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
+
+
+/**
+ * Command for reading graph edges on a connection
+ */
+public class ReadGraphConnectionFilter extends AbstractReadGraphFilter {
+
+ private final String connectionName;
+
+
+ /**
+ * Create a new instance of our command
+ */
+ @Inject
+ public ReadGraphConnectionFilter( final GraphManagerFactory graphManagerFactory, @Assisted final String connectionName ) {
+ super( graphManagerFactory );
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected String getEdgeTypeName() {
+ return getConnectionScopeName( connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
index ec4271a..663c15c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
@@ -24,8 +24,8 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.NoSuchElementException;
-import org.apache.usergrid.corepersistence.command.CommandBuilder;
-import org.apache.usergrid.corepersistence.command.read.entity.EntityLoadCollector;
+import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -82,14 +82,14 @@ public abstract class AbstractGraphQueryExecutor implements QueryExecutor {
//assign them to an iterator. this now uses an internal buffer with backpressure, so we won't load all
// results
//set up our command builder
- final CommandBuilder commandBuilder = new CommandBuilder( applicationScope, sourceId, requestCursor, limit );
+ final DataPipeline dataPipeline = new DataPipeline( applicationScope, sourceId, requestCursor, limit );
- addTraverseCommand( commandBuilder );
+ addTraverseCommand( dataPipeline );
//construct our results to be observed later. This is a cold observable
final Observable<Results> resultsObservable =
- commandBuilder.build( new EntityLoadCollector( entityCollectionManagerFactory, applicationScope ) );
+ dataPipeline.build( new EntityLoadCollectorFilter( entityCollectionManagerFactory, applicationScope ) );
this.observableIterator = resultsObservable.toBlocking().getIterator();
@@ -129,5 +129,5 @@ public abstract class AbstractGraphQueryExecutor implements QueryExecutor {
/**
* Add the traverse command to the graph
*/
- protected abstract void addTraverseCommand( final CommandBuilder commandBuilder );
+ protected abstract void addTraverseCommand( final DataPipeline dataPipeline );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
index 515fc2b..b506c71 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionGraphQueryExecutor.java
@@ -20,8 +20,8 @@
package org.apache.usergrid.corepersistence.results;
-import org.apache.usergrid.corepersistence.command.CommandBuilder;
-import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphCollectionCommand;
+import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -53,8 +53,8 @@ public class CollectionGraphQueryExecutor extends AbstractGraphQueryExecutor {
@Override
- protected void addTraverseCommand( final CommandBuilder commandBuilder ) {
+ protected void addTraverseCommand( final DataPipeline dataPipeline ) {
//set the traverse command from the source Id to the connect name
- commandBuilder.withTraverseCommand( new ReadGraphCollectionCommand( graphManagerFactory, collectionName ) );
+ dataPipeline.withTraverseCommand( new ReadGraphCollectionFilter( graphManagerFactory, collectionName ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
index 61726b7..3dc5cd6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ConnectionGraphQueryExecutor.java
@@ -20,8 +20,8 @@
package org.apache.usergrid.corepersistence.results;
-import org.apache.usergrid.corepersistence.command.CommandBuilder;
-import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphConnectionCommand;
+import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -51,8 +51,8 @@ public class ConnectionGraphQueryExecutor extends AbstractGraphQueryExecutor {
@Override
- protected void addTraverseCommand( final CommandBuilder commandBuilder ) {
+ protected void addTraverseCommand( final DataPipeline dataPipeline ) {
//set the traverse command from the source Id to the connect name
- commandBuilder.withTraverseCommand( new ReadGraphConnectionCommand( graphManagerFactory, connectionName ) );
+ dataPipeline.withTraverseCommand( new ReadGraphConnectionFilter( graphManagerFactory, connectionName ) );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
index d40efc0..99c0cdb 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
@@ -53,7 +53,7 @@ public class SimpleSearchByEdge implements SearchByEdge {
* @param maxTimestamp The maximum timestamp to seek from
* @param last The value to start seeking from. Must be >= this value
*/
- public SimpleSearchByEdge( final Id sourceNode, final String type, final Id targetNode, final long maxTimestamp, final SearchByEdgeType.Order order, final Edge last ) {
+ public SimpleSearchByEdge( final Id sourceNode, final String type, final Id targetNode, final long maxTimestamp, final SearchByEdgeType.Order order, final Optional<Edge> last ) {
ValidationUtils.verifyIdentity(sourceNode);
ValidationUtils.verifyIdentity(targetNode);
@@ -67,7 +67,7 @@ public class SimpleSearchByEdge implements SearchByEdge {
this.type = type;
this.maxTimestamp = maxTimestamp;
this.order = order;
- this.last = Optional.fromNullable(last);
+ this.last = last;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
index 4b73347..9d989a3 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
@@ -25,6 +25,8 @@ import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.SearchByIdType;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
/**
*
@@ -44,7 +46,7 @@ public class SimpleSearchByIdType extends SimpleSearchByEdgeType implements Sear
* @param last The value to start seeking from. Must be >= this value
*/
- public SimpleSearchByIdType( final Id node, final String type, final long maxTimestamp, final Order order, final String idType, final Edge last ) {
+ public SimpleSearchByIdType( final Id node, final String type, final long maxTimestamp, final Order order, final String idType, final Optional<Edge> last ) {
super( node, type, maxTimestamp, order, last );
ValidationUtils.verifyString( idType, "idType" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
index cc0bb51..c81fa58 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
@@ -37,6 +37,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchIdType;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Optional;
+
/**
* Simple class for edge testing generation
@@ -190,7 +192,8 @@ public class EdgeTestUtils {
*/
public static SearchByEdge createGetByEdge( final Id sourceId, final String type, final Id targetId,
final long maxVersion, final Edge last ) {
- return new SimpleSearchByEdge( sourceId, type, targetId, maxVersion, SearchByEdgeType.Order.DESCENDING, last );
+ return new SimpleSearchByEdge( sourceId, type, targetId, maxVersion, SearchByEdgeType.Order.DESCENDING, Optional
+ .of( last ) );
}
//
[13/20] incubator-usergrid git commit: Refactor into commands WIP
Posted by to...@apache.org.
Refactor into commands WIP
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d5978cca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d5978cca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d5978cca
Branch: refs/heads/two-dot-o-dev
Commit: d5978cca07366c43314687081973f48a80b61d8c
Parents: 60617d2
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 14:45:50 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 14:45:50 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 6 +-
.../corepersistence/command/CommandBuilder.java | 10 +-
.../command/read/CollectCommand.java | 36 ---
.../corepersistence/command/read/Collector.java | 36 +++
.../command/read/ReadCommandFactory.java | 81 +++++++
.../AbstractQueryElasticSearchCollector.java | 107 +++++++++
.../QueryCollectionElasticSearchCollector.java | 62 +++++
.../QueryConnectionElasticSearchCollector.java | 62 +++++
.../impl/CollectionRefsVerifier.java | 44 ++++
.../CollectionResultsLoaderFactoryImpl.java | 59 +++++
.../impl/ConnectionRefsVerifier.java | 59 +++++
.../ConnectionResultsLoaderFactoryImpl.java | 65 +++++
.../impl/ElasticSearchQueryExecutor.java | 238 ++++++++++++++++++
.../read/elasticsearch/impl/EntityVerifier.java | 127 ++++++++++
.../elasticsearch/impl/FilteringLoader.java | 226 +++++++++++++++++
.../read/elasticsearch/impl/IdsVerifier.java | 46 ++++
.../read/elasticsearch/impl/ResultsLoader.java | 43 ++++
.../impl/ResultsLoaderFactory.java | 41 ++++
.../elasticsearch/impl/ResultsVerifier.java | 52 ++++
.../elasticsearch/impl/VersionVerifier.java | 85 +++++++
.../read/entity/EntityLoadCollector.java | 138 +++++++++++
.../command/read/entity/EntityLoadCommand.java | 134 -----------
.../read/graph/ReadGraphCollectionCommand.java | 3 +
.../read/graph/ReadGraphConnectionCommand.java | 3 +
.../results/AbstractGraphQueryExecutor.java | 4 +-
.../results/CollectionRefsVerifier.java | 44 ----
.../CollectionResultsLoaderFactoryImpl.java | 59 -----
.../results/ConnectionRefsVerifier.java | 61 -----
.../ConnectionResultsLoaderFactoryImpl.java | 65 -----
.../results/ElasticSearchQueryExecutor.java | 240 -------------------
.../corepersistence/results/EntityVerifier.java | 127 ----------
.../results/FilteringLoader.java | 226 -----------------
.../corepersistence/results/IdsVerifier.java | 46 ----
.../results/ObservableQueryExecutor.java | 76 ++++++
.../corepersistence/results/ResultsLoader.java | 43 ----
.../results/ResultsLoaderFactory.java | 41 ----
.../results/ResultsVerifier.java | 52 ----
.../results/VersionVerifier.java | 85 -------
38 files changed, 1663 insertions(+), 1269 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 86ba300..de7fef1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -32,10 +32,10 @@ import org.springframework.util.Assert;
import org.apache.usergrid.corepersistence.index.AsyncIndexService;
import org.apache.usergrid.corepersistence.results.CollectionGraphQueryExecutor;
-import org.apache.usergrid.corepersistence.results.CollectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.CollectionResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.results.ConnectionGraphQueryExecutor;
-import org.apache.usergrid.corepersistence.results.ConnectionResultsLoaderFactoryImpl;
-import org.apache.usergrid.corepersistence.results.ElasticSearchQueryExecutor;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ElasticSearchQueryExecutor;
import org.apache.usergrid.corepersistence.results.QueryExecutor;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
index de76f50..d9d6a12 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/CommandBuilder.java
@@ -22,7 +22,7 @@ package org.apache.usergrid.corepersistence.command;
import org.apache.usergrid.corepersistence.command.cursor.RequestCursor;
import org.apache.usergrid.corepersistence.command.cursor.ResponseCursor;
-import org.apache.usergrid.corepersistence.command.read.CollectCommand;
+import org.apache.usergrid.corepersistence.command.read.Collector;
import org.apache.usergrid.corepersistence.command.read.Command;
import org.apache.usergrid.corepersistence.command.read.TraverseCommand;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -87,12 +87,12 @@ public class CommandBuilder {
/**
* Build the final collection step, and
*/
- public <T> Observable<T> build( final CollectCommand<T> collectCommand ) {
- setState( collectCommand );
+ public <T> Observable<T> build( final Collector<T> collector ) {
+ setState( collector );
- collectCommand.setLimit( limit );
+ collector.setLimit( limit );
- return currentObservable.compose( collectCommand );
+ return currentObservable.compose( collector );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java
deleted file mode 100644
index c2ad931..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/CollectCommand.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read;
-
-
-/**
- * A command that is used to reduce our stream of results into a final output
- * @param <T>
- */
-public interface CollectCommand<T> extends Command<T>{
-
- /**
- * Set the prefered result size for the command
- * @param resultSize
- */
- void setLimit( final int resultSize );
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java
new file mode 100644
index 0000000..2804db4
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/Collector.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read;
+
+
+/**
+ * A command that is used to reduce our stream of results into a final output
+ * @param <T>
+ */
+public interface Collector<T> extends Command<T>{
+
+ /**
+ * Set the prefered result size for the command
+ * @param limit
+ */
+ void setLimit( final int limit );
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java
new file mode 100644
index 0000000..07f300a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/ReadCommandFactory.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read;
+
+
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.QueryCollectionElasticSearchCollector;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.QueryConnectionElasticSearchCollector;
+import org.apache.usergrid.corepersistence.command.read.entity.EntityLoadCollector;
+import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphCollectionCommand;
+import org.apache.usergrid.corepersistence.command.read.graph.ReadGraphConnectionCommand;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * A factory for generating read commands
+ */
+public interface ReadCommandFactory {
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ * @param applicationScope
+ * @param sourceId
+ * @param collectionName
+ * @return
+ */
+ ReadGraphCollectionCommand readGraphCollectionCommand(final ApplicationScope applicationScope, final Id sourceId, final String collectionName);
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ * @param applicationScope
+ * @param sourceId
+ * @param connectionName
+ * @return
+ */
+ ReadGraphConnectionCommand readGraphConnectionCommand(final ApplicationScope applicationScope, final Id sourceId, final String connectionName);
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ * @param applicationScope
+ * @return
+ */
+ EntityLoadCollector entityLoadCollector(final ApplicationScope applicationScope);
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ * @param applicationScope
+ * @param sourceId
+ * @param collectionName
+ * @return
+ */
+ QueryCollectionElasticSearchCollector queryCollectionElasticSearchCollector(final ApplicationScope applicationScope, final Id sourceId, final String collectionName);
+
+
+ /**
+ * Generate a new instance of the command with the specified parameters
+ * @param applicationScope
+ * @param sourceId
+ * @param connectionName
+ * @return
+ */
+ QueryConnectionElasticSearchCollector queryConnectionElasticSearchCollector(final ApplicationScope applicationScope, final Id sourceId, final String connectionName);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java
new file mode 100644
index 0000000..18cdeb0
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/AbstractQueryElasticSearchCollector.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch;
+
+
+import java.util.Iterator;
+
+import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
+import org.apache.usergrid.corepersistence.command.read.Collector;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ElasticSearchQueryExecutor;
+import org.apache.usergrid.corepersistence.results.QueryExecutor;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+
+/**
+ * A command that will query and load elasticsearch
+ *
+ * On future iteration, this needs to be split into 2 commands 1 that loads the candidate results and validates the
+ * versions then another that will search and load
+ *
+ * TODO, split this into 3 seperate observables
+ *
+ * 1) An observable that emits candidate results
+ * 2) An observable that validates versions by uuid (for Traverse commands)
+ * 3) An observbale that emits Results as a collector (for final commands)
+ */
+public abstract class AbstractQueryElasticSearchCollector extends AbstractCommand<Results, Integer> implements
+ Collector<Results> {
+
+
+ private final ResultsLoaderFactory resultsLoaderFactory;
+ private final ApplicationEntityIndex entityIndex;
+ private final ApplicationScope applicationScope;
+ private final SearchEdge indexScope;
+ private final SearchTypes searchTypes;
+ private final Query query;
+ private int limit;
+
+
+
+ protected AbstractQueryElasticSearchCollector( final ApplicationEntityIndex entityIndex,
+ final ApplicationScope applicationScope, final SearchEdge indexScope,
+ final SearchTypes searchTypes, final Query query ) {
+ this.entityIndex = entityIndex;
+ this.applicationScope = applicationScope;
+ this.indexScope = indexScope;
+ this.searchTypes = searchTypes;
+ this.query = query;
+ this.resultsLoaderFactory = getResultsLoaderFactory();
+ }
+
+
+ @Override
+ public Observable<Results> call( final Observable<Id> idObservable ) {
+ final Iterable<Results> executor =
+ new ElasticSearchQueryExecutor( resultsLoaderFactory, entityIndex, applicationScope, indexScope, searchTypes,
+ query.withLimit( limit ));
+
+ return Observable.from(executor);
+ }
+
+
+ /**
+ * Get the results loader factor
+ * @return
+ */
+ protected abstract ResultsLoaderFactory getResultsLoaderFactory();
+
+ @Override
+ protected Class<Integer> getCursorClass() {
+ return Integer.class;
+ }
+
+
+ @Override
+ public void setLimit( final int limit ) {
+ this.limit = limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java
new file mode 100644
index 0000000..2bbcc48
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryCollectionElasticSearchCollector.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+
+import com.google.inject.Inject;
+
+
+/**
+ * Command for querying connections
+ */
+public class QueryCollectionElasticSearchCollector extends AbstractQueryElasticSearchCollector {
+
+ private final ManagerCache managerCache;
+ private final EntityRef headEntity;
+ private final String connectionName;
+
+ @Inject
+ protected QueryCollectionElasticSearchCollector( final ApplicationEntityIndex entityIndex,
+ final ApplicationScope applicationScope,
+ final SearchEdge indexScope, final SearchTypes searchTypes,
+ final Query query, final ManagerCache managerCache,
+ final EntityRef headEntity, final String connectionName ) {
+ super( entityIndex, applicationScope, indexScope, searchTypes, query );
+ this.managerCache = managerCache;
+ this.headEntity = headEntity;
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected ResultsLoaderFactory getResultsLoaderFactory() {
+ return new ConnectionResultsLoaderFactoryImpl( managerCache, headEntity, connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java
new file mode 100644
index 0000000..0e2f221
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/QueryConnectionElasticSearchCollector.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.command.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+
+import com.google.inject.Inject;
+
+
+/**
+ * Command for querying connections
+ */
+public class QueryConnectionElasticSearchCollector extends AbstractQueryElasticSearchCollector {
+
+ private final ManagerCache managerCache;
+ private final EntityRef headEntity;
+ private final String connectionName;
+
+ @Inject
+ protected QueryConnectionElasticSearchCollector( final ApplicationEntityIndex entityIndex,
+ final ApplicationScope applicationScope,
+ final SearchEdge indexScope, final SearchTypes searchTypes,
+ final Query query, final ManagerCache managerCache,
+ final EntityRef headEntity, final String connectionName ) {
+ super( entityIndex, applicationScope, indexScope, searchTypes, query );
+ this.managerCache = managerCache;
+ this.headEntity = headEntity;
+ this.connectionName = connectionName;
+ }
+
+
+ @Override
+ protected ResultsLoaderFactory getResultsLoaderFactory() {
+ return new ConnectionResultsLoaderFactoryImpl( managerCache, headEntity, connectionName );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java
new file mode 100644
index 0000000..4b67ae8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionRefsVerifier.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class CollectionRefsVerifier extends VersionVerifier {
+
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+ List<EntityRef> refs = new ArrayList<EntityRef>(ids.size());
+ for ( Id id : ids ) {
+ refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+ }
+ return Results.fromRefList( refs );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..f9dd4e1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public class CollectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+ private final ManagerCache managerCache;
+
+
+ public CollectionResultsLoaderFactoryImpl( final ManagerCache managerCache ) {
+ this.managerCache = managerCache;
+ }
+
+
+ @Override
+ public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope, final Query.Level resultsLevel ) {
+
+ ResultsVerifier verifier;
+
+ if ( resultsLevel == Query.Level.REFS ) {
+ verifier = new CollectionRefsVerifier();
+ }
+ else if ( resultsLevel == Query.Level.IDS ) {
+ verifier = new IdsVerifier();
+ }
+ else {
+ verifier = new EntityVerifier(Query.MAX_LIMIT);
+ }
+
+ return new FilteringLoader( managerCache, verifier, applicationScope, scope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java
new file mode 100644
index 0000000..bffb53a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionRefsVerifier.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+
+
+/**
+ * Verifier for creating connections
+ */
+public class ConnectionRefsVerifier extends VersionVerifier {
+
+
+ private final EntityRef ownerId;
+ private final String connectionType;
+
+
+ public ConnectionRefsVerifier( final EntityRef ownerId, final String connectionType ) {
+ this.ownerId = ownerId;
+ this.connectionType = connectionType;
+ }
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+ List<ConnectionRef> refs = new ArrayList<>();
+ for ( Id id : ids ) {
+ refs.add( new ConnectionRefImpl( ownerId, connectionType, ref(id.getType(), id.getUuid()) ));
+ }
+
+ return Results.fromConnections( refs );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..707e933
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public class ConnectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+ private final ManagerCache managerCache;
+ private final EntityRef ownerId;
+ private final String connectionType;
+
+
+ public ConnectionResultsLoaderFactoryImpl( final ManagerCache managerCache, final EntityRef ownerId,
+ final String connectionType ) {
+ this.managerCache = managerCache;
+ this.ownerId = ownerId;
+ this.connectionType = connectionType;
+ }
+
+
+ @Override
+ public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope, final Query.Level resultsLevel ) {
+
+ ResultsVerifier verifier;
+
+ if ( resultsLevel == Query.Level.REFS ) {
+ verifier = new ConnectionRefsVerifier( ownerId, connectionType );
+ }
+ else if ( resultsLevel == Query.Level.IDS ) {
+ verifier = new ConnectionRefsVerifier( ownerId, connectionType );;
+ }
+ else {
+ verifier = new EntityVerifier(Query.MAX_LIMIT);
+ }
+
+ return new FilteringLoader( managerCache, verifier, applicationScope, scope );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
new file mode 100644
index 0000000..c7e8f56
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ElasticSearchQueryExecutor.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.usergrid.corepersistence.results.QueryExecutor;
+import org.apache.usergrid.persistence.index.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+public class ElasticSearchQueryExecutor implements Iterable<Results>, Iterator<Results> {
+
+ private static final Logger logger = LoggerFactory.getLogger( ElasticSearchQueryExecutor.class );
+
+ private final ResultsLoaderFactory resultsLoaderFactory;
+
+ private final ApplicationScope applicationScope;
+
+ private final ApplicationEntityIndex entityIndex;
+
+ private final SearchEdge indexScope;
+
+ private final SearchTypes types;
+
+ private final Query query;
+
+
+ private Results currentResults;
+
+ private boolean moreToLoad = true;
+
+
+ public ElasticSearchQueryExecutor( final ResultsLoaderFactory resultsLoaderFactory, final ApplicationEntityIndex entityIndex,
+ final ApplicationScope applicationScope, final SearchEdge indexScope,
+ final SearchTypes types, final Query query ) {
+ this.resultsLoaderFactory = resultsLoaderFactory;
+ this.applicationScope = applicationScope;
+ this.entityIndex = entityIndex;
+ this.indexScope = indexScope;
+ this.types = types;
+
+ //we must deep copy the query passed. Otherwise we will modify it's state with cursors. Won't fix, not relevant
+ //once we start subscribing to streams.
+ this.query = new Query(query);
+ }
+
+
+ @Override
+ public Iterator<Results> iterator() {
+ return this;
+ }
+
+
+ private void loadNextPage() {
+ // Because of possible stale entities, which are filtered out by buildResults(),
+ // we loop until the we've got enough results to satisfy the query limit.
+
+ final int maxQueries = 10; // max re-queries to satisfy query limit
+
+ final int originalLimit = query.getLimit();
+
+ Results results = null;
+ int queryCount = 0;
+
+
+ CandidateResults crs = null;
+
+ while ( queryCount++ < maxQueries ) {
+
+ crs = getCandidateResults( query );
+
+
+ logger.debug( "Calling build results with crs {}", crs );
+ results = buildResults( indexScope, query, crs );
+
+ /**
+ * In an edge case where we delete stale entities, we could potentially get less results than expected.
+ * This will only occur once during the repair phase.
+ * We need to ensure that we short circuit before we overflow the requested limit during a repair.
+ */
+ if ( crs.isEmpty() || !crs.hasOffset() || results.size() > 0 ) { // no results, no cursor, can't get more
+ break;
+ }
+
+
+ //we didn't load anything, but there was a cursor, this means a read repair occured. We have to short
+ //circuit to avoid over returning the result set
+
+
+ // need to query for more
+ // ask for just what we need to satisfy, don't want to exceed limit
+ query.setOffsetFromCursor(results.getCursor());
+ query.setLimit( originalLimit - results.size() );
+
+ logger.warn( "Satisfy query limit {}, new limit {} query count {}", new Object[] {
+ originalLimit, query.getLimit(), queryCount
+ } );
+ }
+
+ //now set our cursor if we have one for the next iteration
+ if ( results.hasCursor() ) {
+ query.setOffsetFromCursor(results.getCursor());
+ moreToLoad = true;
+ }
+
+ else {
+ moreToLoad = false;
+ }
+
+
+ //set our select subjects into our query if provided
+ if(crs != null){
+ query.setSelectSubjects( crs.getGetFieldMappings() );
+ }
+
+
+ //set our current results and the flag
+ this.currentResults = results;
+ }
+
+
+ /**
+ * Get the candidates or load the cursor, whichever we require
+ * @param query
+ * @return
+ */
+ private CandidateResults getCandidateResults(final Query query){
+ final Optional<Integer> cursor = query.getOffset();
+ final String queryToExecute = query.getQl().or("select *");
+
+ CandidateResults results = cursor.isPresent()
+ ? entityIndex.search( indexScope, types, queryToExecute, query.getLimit() , cursor.get())
+ : entityIndex.search( indexScope, types, queryToExecute, query.getLimit());
+
+ return results;
+ }
+
+
+ /**
+ * Build results from a set of candidates, and discard those that represent stale indexes.
+ *
+ * @param query Query that was executed
+ * @param crs Candidates to be considered for results
+ */
+ private Results buildResults( final SearchEdge indexScope, final Query query, final CandidateResults crs ) {
+
+ logger.debug( "buildResults() from {} candidates", crs.size() );
+
+ //get an instance of our results loader
+ final ResultsLoader resultsLoader =
+ this.resultsLoaderFactory.getLoader( applicationScope, indexScope, query.getResultsLevel() );
+
+ //load the results
+ final Results results = resultsLoader.loadResults(crs);
+
+ //signal for post processing
+ resultsLoader.postProcess();
+
+ //set offset into query
+ if(crs.getOffset().isPresent()) {
+ query.setOffset(crs.getOffset().get());
+ }else{
+ query.clearOffset();
+ }
+ results.setCursorFromOffset( query.getOffset() );
+
+ logger.debug( "Returning results size {}", results.size() );
+
+ return results;
+ }
+
+
+ @Override
+ public boolean hasNext() {
+
+ //we've tried to load and it's empty and we have more to load, load the next page
+ if ( currentResults == null ) {
+ //there's nothing left to load, nothing to do
+ if ( !moreToLoad ) {
+ return false;
+ }
+
+ //load the page
+
+ loadNextPage();
+ }
+
+
+ //see if our current results are not null
+ return currentResults != null;
+ }
+
+
+ @Override
+ public Results next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No more results present" );
+ }
+
+ final Results toReturn = currentResults;
+
+ currentResults = null;
+
+ return toReturn;
+ }
+
+ @Override
+ public void remove() {
+ throw new RuntimeException("Remove not implemented!!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java
new file mode 100644
index 0000000..7be8aa4
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/EntityVerifier.java
@@ -0,0 +1,127 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Optional;
+
+
+/**
+ * A loader that verifies versions are correct in cassandra and match elasticsearch
+ */
+public class EntityVerifier implements ResultsVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
+
+ private EntitySet ids;
+
+ private Map<Id, org.apache.usergrid.persistence.model.entity.Entity> entityMapping;
+
+
+ public EntityVerifier( final int maxSize ) {
+ this.entityMapping = new HashMap<>( maxSize );
+ }
+
+
+ @Override
+ public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
+ ids = ecm.load( idsToLoad ).toBlocking().last();
+ logger.debug("loadResults() asked for {} ids and got {}", idsToLoad.size(), ids.size());
+ }
+
+
+ @Override
+ public boolean isValid( final CandidateResult candidateResult ) {
+ final Id entityId = candidateResult.getId();
+
+ final MvccEntity savedEntity = ids.getEntity( entityId );
+
+ //version wasn't found deindex
+ if ( savedEntity == null ) {
+ logger.warn( "Version for Entity {}:{} not found", entityId.getType(), entityId.getUuid() );
+ return false;
+ }
+
+ final UUID candidateVersion = candidateResult.getVersion();
+ final UUID savedVersion = savedEntity.getVersion();
+
+ if ( UUIDComparator.staticCompare( savedVersion, candidateVersion ) > 0 ) {
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), candidateVersion, savedEntity
+ } );
+
+ return false;
+ }
+
+
+ final Optional<org.apache.usergrid.persistence.model.entity.Entity> entity = savedEntity.getEntity();
+
+ if ( !entity.isPresent() ) {
+ logger.warn( "Entity uuid:{} version v:{} is deleted but indexed, this is a bug ",
+ entityId.getUuid(), savedEntity.getEntity() );
+ return false;
+ }
+
+ entityMapping.put( entityId, entity.get() );
+
+ return true;
+ }
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+
+ final List<Entity> ugEntities = new ArrayList<>( ids.size() );
+
+ for ( final Id id : ids ) {
+ final org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityMapping.get( id );
+
+ Entity entity = EntityFactory.newEntity( id.getUuid(), id.getType() );
+
+ Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
+ entity.addProperties( entityMap );
+ ugEntities.add( entity );
+ }
+
+ return Results.fromEntities( ugEntities );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java
new file mode 100644
index 0000000..c0afe92
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/FilteringLoader.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.ManagerCache;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.index.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+
+
+public class FilteringLoader implements ResultsLoader {
+
+ private static final Logger logger = LoggerFactory.getLogger( FilteringLoader.class );
+
+ private final ManagerCache managerCache;
+ private final ResultsVerifier resultsVerifier;
+ private final ApplicationScope applicationScope;
+ private final SearchEdge indexScope;
+ private final EntityIndexBatch indexBatch;
+
+
+ /**
+ * Create an instance of a filter loader
+ *
+ * @param managerCache The manager cache to load
+ * @param resultsVerifier The verifier to verify the candidate results
+ * @param applicationScope The application scope to perform the load
+ * @param indexScope The index scope used in the search
+ */
+ protected FilteringLoader( final ManagerCache managerCache, final ResultsVerifier resultsVerifier,
+ final ApplicationScope applicationScope, final SearchEdge indexScope ) {
+
+ this.managerCache = managerCache;
+ this.resultsVerifier = resultsVerifier;
+ this.applicationScope = applicationScope;
+ this.indexScope = indexScope;
+
+ final ApplicationEntityIndex index = managerCache.getEntityIndex( applicationScope );
+
+ indexBatch = index.createBatch();
+ }
+
+
+ @Override
+ public Results loadResults( final CandidateResults crs ) {
+
+
+ if ( crs.size() == 0 ) {
+ return new Results();
+ }
+
+
+ // For each entity, holds the index it appears in our candidates for keeping ordering correct
+ final Map<Id, Integer> orderIndex = new HashMap<>( crs.size() );
+
+ // Maps the entity ids to our candidates
+ final Map<Id, CandidateResult> maxCandidateMapping = new HashMap<>( crs.size() );
+
+
+ final Iterator<CandidateResult> iter = crs.iterator();
+
+
+ // TODO, in this case we're "optimizing" due to the limitations of collection scope.
+ // Perhaps we should change the API to just be an application, then an "owner" scope?
+
+ // Go through the candidates and group them by scope for more efficient retrieval.
+ // Also remove duplicates before we even make a network call
+ for ( int i = 0; iter.hasNext(); i++ ) {
+
+ final CandidateResult currentCandidate = iter.next();
+
+ final Id entityId = currentCandidate.getId();
+
+ //check if we've seen this candidate by id
+ final CandidateResult previousMax = maxCandidateMapping.get( entityId );
+
+ //its not been seen, save it
+ if ( previousMax == null ) {
+ maxCandidateMapping.put( entityId, currentCandidate );
+ orderIndex.put( entityId, i );
+ continue;
+ }
+
+ //we have seen it, compare them
+
+ final UUID previousMaxVersion = previousMax.getVersion();
+
+ final UUID currentVersion = currentCandidate.getVersion();
+
+
+ final CandidateResult toRemove;
+ final CandidateResult toKeep;
+
+ //current is newer than previous. Remove previous and keep current
+ if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
+ toRemove = previousMax;
+ toKeep = currentCandidate;
+ }
+ //previously seen value is newer than current. Remove the current and keep the previously seen value
+ else {
+ toRemove = currentCandidate;
+ toKeep = previousMax;
+ }
+
+ //this is a newer version, we know we already have a stale entity, add it to be cleaned up
+
+
+ //de-index it
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), toRemove.getVersion(), toKeep.getVersion()
+ } );
+
+ //deindex this document, and remove the previous maxVersion
+ //we have to deindex this from our ownerId, since this is what gave us the reference
+ indexBatch.deindex( indexScope, toRemove );
+
+
+ //TODO, fire the entity repair cleanup task here instead of de-indexing
+
+ //replace the value with a more current version
+ maxCandidateMapping.put( entityId, toKeep );
+ orderIndex.put( entityId, i );
+ }
+
+
+ //now everything is ordered, and older versions are removed. Batch fetch versions to verify
+ // existence and correct versions
+
+ final TreeMap<Integer, Id> sortedResults = new TreeMap<>();
+
+
+ final Collection<Id> idsToLoad =
+ Collections2.transform( maxCandidateMapping.values(), new Function<CandidateResult, Id>() {
+ @Nullable
+ @Override
+ public Id apply( @Nullable final CandidateResult input ) {
+ //NOTE this is never null, we won't need to check
+ return input.getId();
+ }
+ } );
+
+
+ //now using the scope, load the collection
+
+ // Get the collection scope and batch load all the versions. We put all entities in
+ // app/app for easy retrieval/ unless persistence changes, we never want to read from
+ // any scope other than the app, app, scope name scope
+ // final CollectionScope collScope = new CollectionScopeImpl(
+ // applicationScope.getApplication(), applicationScope.getApplication(), scopeName);
+
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( applicationScope );
+
+
+ //load the results into the loader for this scope for validation
+ resultsVerifier.loadResults( idsToLoad, ecm );
+
+ //now let the loader validate each candidate. For instance, the "max" in this candidate
+ //could still be a stale result, so it needs validated
+ for ( final Id requestedId : idsToLoad ) {
+
+ final CandidateResult cr = maxCandidateMapping.get( requestedId );
+
+ //ask the loader if this is valid, if not discard it and de-index it
+ if ( !resultsVerifier.isValid( cr ) ) {
+ indexBatch.deindex( indexScope, cr );
+ continue;
+ }
+
+ //if we get here we're good, we need to add this to our results
+ final int candidateIndex = orderIndex.get( requestedId );
+
+ sortedResults.put( candidateIndex, requestedId );
+ }
+
+
+ // NOTE DO NOT execute the batch here.
+ // It changes the results and we need consistent paging until we aggregate all results
+ return resultsVerifier.getResults( sortedResults.values() );
+ }
+
+
+ @Override
+ public void postProcess() {
+ this.indexBatch.execute();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java
new file mode 100644
index 0000000..4d2bd55
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/IdsVerifier.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class IdsVerifier extends VersionVerifier {
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+
+ final List<UUID> returnIds = new ArrayList<>( ids.size() );
+
+ for ( final Id id : ids ) {
+ returnIds.add( id.getUuid() );
+ }
+
+
+ return Results.fromIdList( returnIds );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java
new file mode 100644
index 0000000..fa0e71f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoader.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.CandidateResults;
+
+
+/**
+ * Interface for loading results
+ */
+public interface ResultsLoader {
+
+ /**
+ * Using the candidate results, load our results. Should filter stale results
+ * @param crs The candidate result set
+ * @return Results. Null safe, but may be empty
+ */
+ public Results loadResults( final CandidateResults crs);
+
+ /**
+ * Post process the load operation
+ */
+ public void postProcess();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java
new file mode 100644
index 0000000..14db80e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsLoaderFactory.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public interface ResultsLoaderFactory {
+
+ /**
+ * Get the loader for results
+ * @param applicationScope The application scope used to load results
+ * @param indexScope The index scope used in the search
+ * @param
+ */
+ ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge indexScope,
+ final Query.Level resultsLevel );
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java
new file mode 100644
index 0000000..68515fa
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/ResultsVerifier.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.Collection;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public interface ResultsVerifier {
+
+ /**
+ * Load all the candidate ides for verification
+ * @param ids The Id's to load
+ * @param ecm The entity collection manager
+ */
+ public void loadResults(Collection<Id> ids, EntityCollectionManager ecm);
+
+ /**
+ * Return true if the candidate result is a valid result that should be retained. If it should
+ * not it should also be removed from the list of possible return values in this loader
+ * @param candidateResult
+ */
+ public boolean isValid(CandidateResult candidateResult);
+
+
+ /**
+ * Load the result set with the given ids
+ * @return
+ */
+ public Results getResults(Collection<Id> ids);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
new file mode 100644
index 0000000..58e4296
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
+
+
+import java.util.Collection;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.index.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+
+
+/**
+ * A loader that verifies versions are correct in Cassandra and match ElasticSearch
+ */
+public abstract class VersionVerifier implements ResultsVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( VersionVerifier.class );
+
+ private VersionSet ids;
+
+
+ @Override
+ public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
+ ids = ecm.getLatestVersion( idsToLoad ).toBlocking().last();
+ }
+
+
+ @Override
+ public boolean isValid( final CandidateResult candidateResult ) {
+ final Id entityId = candidateResult.getId();
+
+ final MvccLogEntry version = ids.getMaxVersion( entityId );
+
+ //version wasn't found ,deindex
+ if ( version == null ) {
+ logger.warn( "Version for Entity {}:{} not found",
+ entityId.getUuid(), entityId.getUuid() );
+
+ return false;
+ }
+
+ final UUID savedVersion = version.getVersion();
+
+ if ( UUIDComparator.staticCompare( savedVersion, candidateResult.getVersion() ) > 0 ) {
+ logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] {
+ entityId.getUuid(),
+ entityId.getType(),
+ candidateResult.getVersion(),
+ savedVersion
+ } );
+
+ return false;
+ }
+
+ return true;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
new file mode 100644
index 0000000..1504f26
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.command.read.entity;
+
+
+import java.io.Serializable;
+import java.util.Map;
+
+import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
+import org.apache.usergrid.corepersistence.command.read.Collector;
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Loads entities from a set of Ids.
+ *
+ * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
+ */
+public class EntityLoadCollector extends AbstractCommand<Results, Serializable> implements Collector<Results> {
+
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+ //TODO get rid of this when merged into 2.0 dev
+ private final ApplicationScope applicationScope;
+ private int resultSize;
+
+
+ public EntityLoadCollector( final EntityCollectionManagerFactory entityCollectionManagerFactory,
+ final ApplicationScope applicationScope ) {
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.applicationScope = applicationScope;
+ }
+
+
+ @Override
+ protected Class<Serializable> getCursorClass() {
+ return null;
+ }
+
+
+ @Override
+ public Observable<Results> call( final Observable<Id> observable ) {
+
+
+ /**
+ * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
+ * objects
+ */
+
+ final EntityCollectionManager entityCollectionManager =
+ entityCollectionManagerFactory.createCollectionManager( applicationScope );
+
+ final Observable<EntitySet> entitySetObservable = observable.buffer( resultSize ).flatMap(
+ bufferedIds -> Observable.just( bufferedIds ).flatMap( ids -> entityCollectionManager.load( ids ) ) );
+
+
+ final Observable<Results> resultsObservable = entitySetObservable
+
+ .flatMap( entitySet -> {
+
+ //get our entites and filter missing ones, then collect them into a results object
+ final Observable<MvccEntity> mvccEntityObservable = Observable.from( entitySet.getEntities() );
+
+ //convert them to our old entity model, then filter nulls, meaning they weren't found
+ return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter( entity -> entity == null )
+
+ //convert them to a list, then map them into results
+ .toList().map( entities -> {
+ final Results results = Results.fromEntities( entities );
+ results.setCursor( generateCursor() );
+
+ return results;
+ } )
+ //if no results are present, return an empty results
+ .singleOrDefault( new Results( ) );
+ } );
+
+
+ return resultsObservable;
+ }
+
+ /**
+ * Map a new cp entity to an old entity. May be null if not present
+ */
+
+
+ private org.apache.usergrid.persistence.Entity mapEntity( final MvccEntity mvccEntity ) {
+ if ( !mvccEntity.getEntity().isPresent() ) {
+ return null;
+ }
+
+
+ final Entity cpEntity = mvccEntity.getEntity().get();
+ final Id entityId = cpEntity.getId();
+
+ org.apache.usergrid.persistence.Entity entity =
+ EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
+
+ Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
+ entity.addProperties( entityMap );
+
+ return entity;
+ }
+
+
+ @Override
+ public void setLimit( final int limit ) {
+ this.resultSize = limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
deleted file mode 100644
index 08193ef..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCommand.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.entity;
-
-
-import java.io.Serializable;
-import java.util.Map;
-
-import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
-import org.apache.usergrid.corepersistence.command.read.CollectCommand;
-import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Loads entities from a set of Ids.
- *
- * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
- */
-public class EntityLoadCommand extends AbstractCommand<Results, Serializable> implements CollectCommand<Results> {
-
- private final EntityCollectionManagerFactory entityCollectionManagerFactory;
-
- //TODO get rid of this when merged into 2.0 dev
- private final ApplicationScope applicationScope;
- private int resultSize;
-
-
- public EntityLoadCommand( final EntityCollectionManagerFactory entityCollectionManagerFactory,
- final ApplicationScope applicationScope ) {
- this.entityCollectionManagerFactory = entityCollectionManagerFactory;
- this.applicationScope = applicationScope;
- }
-
-
- @Override
- protected Class<Serializable> getCursorClass() {
- return null;
- }
-
-
- @Override
- public Observable<Results> call( final Observable<Id> observable ) {
-
-
- /**
- * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results
- * objects
- */
-
- final EntityCollectionManager entityCollectionManager =
- entityCollectionManagerFactory.createCollectionManager( applicationScope );
-
- final Observable<EntitySet> entitySetObservable = observable.buffer( resultSize ).flatMap(
- bufferedIds -> Observable.just( bufferedIds ).flatMap( ids -> entityCollectionManager.load( ids ) ) );
-
-
- return entitySetObservable
-
- .flatMap( entitySet -> {
-
- //get our entites and filter missing ones, then collect them into a results object
- final Observable<MvccEntity> mvccEntityObservable = Observable.from( entitySet.getEntities() );
-
- //convert them to our old entity model, then filter nulls, meaning they weren't found
- return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter(
- entity -> entity == null )
-
- //convert them to a list, then map them into results
- .toList().map( entities -> {
- final Results results = Results.fromEntities( entities );
- results.setCursor( generateCursor() );
-
- return results;
- } );
- } );
- }
-
- /**
- * Map a new cp entity to an old entity. May be null if not present
- */
-
-
- private org.apache.usergrid.persistence.Entity mapEntity( final MvccEntity mvccEntity ) {
- if ( !mvccEntity.getEntity().isPresent() ) {
- return null;
- }
-
-
- final Entity cpEntity = mvccEntity.getEntity().get();
- final Id entityId = cpEntity.getId();
-
- org.apache.usergrid.persistence.Entity entity =
- EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
- entity.addProperties( entityMap );
-
- return entity;
- }
-
-
- @Override
- public void setLimit( final int resultSize ) {
- this.resultSize = resultSize;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
index 9ccb969..3b6633b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.corepersistence.command.read.graph;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import com.google.inject.Inject;
+
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
@@ -36,6 +38,7 @@ public class ReadGraphCollectionCommand extends AbstractReadGraphCommand {
/**
* Create a new instance of our command
*/
+ @Inject
public ReadGraphCollectionCommand( final GraphManagerFactory graphManagerFactory, final String collectionName ) {
super( graphManagerFactory );
this.collectionName = collectionName;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
index adebd45..26b30e7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.corepersistence.command.read.graph;
import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+import com.google.inject.Inject;
+
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
@@ -36,6 +38,7 @@ public class ReadGraphConnectionCommand extends AbstractReadGraphCommand {
/**
* Create a new instance of our command
*/
+ @Inject
public ReadGraphConnectionCommand( final GraphManagerFactory graphManagerFactory, final String connectionName ) {
super( graphManagerFactory );
this.connectionName = connectionName;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5978cca/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
index cf0764e..ec4271a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractGraphQueryExecutor.java
@@ -25,7 +25,7 @@ import java.util.Iterator;
import java.util.NoSuchElementException;
import org.apache.usergrid.corepersistence.command.CommandBuilder;
-import org.apache.usergrid.corepersistence.command.read.entity.EntityLoadCommand;
+import org.apache.usergrid.corepersistence.command.read.entity.EntityLoadCollector;
import org.apache.usergrid.persistence.EntityRef;
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -89,7 +89,7 @@ public abstract class AbstractGraphQueryExecutor implements QueryExecutor {
//construct our results to be observed later. This is a cold observable
final Observable<Results> resultsObservable =
- commandBuilder.build( new EntityLoadCommand( entityCollectionManagerFactory, applicationScope ) );
+ commandBuilder.build( new EntityLoadCollector( entityCollectionManagerFactory, applicationScope ) );
this.observableIterator = resultsObservable.toBlocking().getIterator();
[20/20] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
USERGRID-593
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-593
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ff2b3f14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ff2b3f14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ff2b3f14
Branch: refs/heads/two-dot-o-dev
Commit: ff2b3f142f5406e01eef1e48780d872d0f20d499
Parents: 223c4b3 bd3d939
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 27 11:38:34 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 27 11:38:34 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 4 +-
.../corepersistence/index/IndexServiceImpl.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 45 ++++++++-----
.../persistence/core/future/BetterFuture.java | 68 --------------------
.../core/future/FutureObservable.java | 53 +++++++++++++++
.../persistence/index/EntityIndexBatch.java | 5 +-
.../index/impl/EsEntityIndexBatchImpl.java | 7 +-
.../index/impl/EsEntityIndexImpl.java | 6 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 8 +--
.../index/impl/IndexBufferConsumer.java | 4 +-
.../index/impl/IndexOperationMessage.java | 19 ++----
.../persistence/index/impl/EntityIndexTest.java | 36 +++++------
.../persistence/index/impl/GeoPagingTest.java | 2 +-
13 files changed, 126 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ff2b3f14/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
[09/20] incubator-usergrid git commit: Merge branch 'two-dot-o-dev'
into USERGRID-593
Posted by to...@apache.org.
Merge branch 'two-dot-o-dev' into USERGRID-593
# Conflicts:
# stack/core/pom.xml
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/ApplicationObservable.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesFromSourceObservable.java
# stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/EdgesToTargetObservable.java
# stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
# stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
# stack/corepersistence/pom.xml
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
# stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
# stack/pom.xml
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ec3a710a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ec3a710a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ec3a710a
Branch: refs/heads/two-dot-o-dev
Commit: ec3a710adf5fd8c88e27e3c248d50c45c7164d18
Parents: 6904c9d 852c974
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Apr 24 09:59:57 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Apr 24 09:59:57 2015 -0600
----------------------------------------------------------------------
portal/index-template.html | 2 +-
portal/js/data/data-controller.js | 58 +-
portal/js/data/data.html | 36 +-
portal/js/global/ug-service.js | 30 +-
portal/js/login/login-controller.js | 15 +-
.../Usergrid.Notifications/BackgroundTask.cs | 57 +
.../Usergrid.Notifications/Client/PushClient.cs | 12 +-
.../Usergrid.Notifications/MainPage.xaml.cs | 12 +-
.../Usergrid.Notifications/MyBackgroundTask.cs | 18 +
.../Usergrid.Notifications/Package.appxmanifest | 8 +
.../Usergrid.Notifications.csproj | 1 +
.../Usergrid.Notifications.sln | 40 +
.../main/dist/init_instance/init_rest_server.sh | 6 +-
.../main/resources/usergrid-default.properties | 5 +-
stack/core/pom.xml | 46 +-
.../batch/service/JobSchedulerService.java | 28 +-
.../batch/service/SchedulerService.java | 2 +-
.../batch/service/SchedulerServiceImpl.java | 23 +-
.../corepersistence/ApplicationIdCache.java | 51 +
.../ApplicationIdCacheFactory.java | 43 +
.../corepersistence/ApplicationIdCacheFig.java | 43 +
.../corepersistence/ApplicationIdCacheImpl.java | 146 +
.../usergrid/corepersistence/CoreModule.java | 137 +-
.../corepersistence/CpEntityManager.java | 411 +--
.../corepersistence/CpEntityManagerFactory.java | 579 ++--
.../corepersistence/CpManagerCache.java | 65 +-
.../corepersistence/CpQueryProcessor.java | 94 -
.../corepersistence/CpRelationManager.java | 1842 ++---------
.../usergrid/corepersistence/CpSetup.java | 32 +-
.../usergrid/corepersistence/CpWalker.java | 83 +-
.../usergrid/corepersistence/GuiceFactory.java | 75 +-
.../usergrid/corepersistence/ManagerCache.java | 17 +-
.../corepersistence/OrgApplicationCache.java | 67 -
.../OrgApplicationCacheImpl.java | 181 --
.../events/EntityDeletedHandler.java | 43 +-
.../events/EntityVersionCreatedHandler.java | 22 +-
.../events/EntityVersionDeletedHandler.java | 144 +-
.../index/AsyncIndexProvider.java | 119 +
.../index/AsyncIndexService.java | 42 +
.../index/InMemoryAsyncIndexService.java | 88 +
.../corepersistence/index/IndexEntityEvent.java | 75 +
.../index/IndexProcessorFig.java | 86 +
.../corepersistence/index/IndexService.java | 52 +
.../corepersistence/index/IndexServiceImpl.java | 172 +
.../corepersistence/index/ReIndexAction.java | 33 +
.../corepersistence/index/ReIndexService.java | 89 +
.../index/ReIndexServiceImpl.java | 129 +
.../index/SQSAsyncIndexService.java | 325 ++
.../migration/AppInfoMigrationPlugin.java | 291 ++
.../migration/AppInfoVersions.java | 40 +
.../migration/CoreDataVersions.java | 45 +
.../migration/CoreMigration.java | 53 +
.../migration/CoreMigrationPlugin.java | 64 +
.../migration/EntityDataMigration.java | 147 -
.../migration/EntityTypeMappingMigration.java | 69 +-
.../migration/GraphShardVersionMigration.java | 152 -
.../migration/MigrationModuleVersionPlugin.java | 145 +
.../migration/MigrationSystemVersions.java | 43 +
.../corepersistence/migration/Versions.java | 14 -
.../CollectionResultsLoaderFactoryImpl.java | 6 +-
.../ConnectionResultsLoaderFactoryImpl.java | 6 +-
.../results/ElasticSearchQueryExecutor.java | 92 +-
.../corepersistence/results/EntityVerifier.java | 2 +-
.../results/FilteringLoader.java | 122 +-
.../corepersistence/results/ResultsLoader.java | 2 +-
.../results/ResultsLoaderFactory.java | 7 +-
.../results/ResultsVerifier.java | 4 +-
.../results/VersionVerifier.java | 12 +-
.../rx/AllEntitiesInSystemObservable.java | 101 -
.../rx/ApplicationObservable.java | 130 -
.../rx/EdgesFromSourceObservable.java | 65 -
.../rx/EdgesToTargetObservable.java | 65 -
.../corepersistence/rx/TargetIdObservable.java | 66 -
.../rx/impl/AbstractGraphVisitorImpl.java | 65 +
.../rx/impl/AllApplicationsObservable.java | 43 +
.../rx/impl/AllApplicationsObservableImpl.java | 133 +
.../rx/impl/AllEntitiesInSystemImpl.java | 53 +
.../rx/impl/AllEntityIdsObservable.java | 52 +
.../rx/impl/AllEntityIdsObservableImpl.java | 92 +
.../rx/impl/AllNodesInGraphImpl.java | 54 +
.../corepersistence/rx/impl/EdgeScope.java | 51 +
.../corepersistence/util/CpEntityMapUtils.java | 19 +-
.../corepersistence/util/CpNamingUtils.java | 247 +-
.../util/SerializableMapper.java | 91 +
.../usergrid/exception/ConflictException.java | 36 +
.../main/java/org/apache/usergrid/mq/Query.java | 2 +-
.../org/apache/usergrid/mq/QueryProcessor.java | 210 +-
.../apache/usergrid/mq/cassandra/QueuesCF.java | 5 -
.../usergrid/persistence/CounterQuery.java | 2 +-
.../usergrid/persistence/EntityManager.java | 51 +-
.../persistence/EntityManagerFactory.java | 134 +-
.../persistence/EntityPropertyComparator.java | 56 -
.../persistence/GuiceAdapterBeanFactory.java | 114 +
.../persistence/MultiQueryIterator.java | 4 +-
.../persistence/ObservableIterator.java | 1 +
.../persistence/PagingResultsIterator.java | 4 +-
.../apache/usergrid/persistence/PathQuery.java | 6 +-
.../usergrid/persistence/PersistenceModule.java | 66 +
.../org/apache/usergrid/persistence/Query.java | 912 ++++++
.../apache/usergrid/persistence/QueryUtils.java | 61 +-
.../usergrid/persistence/RelationManager.java | 7 +-
.../apache/usergrid/persistence/Results.java | 70 +-
.../org/apache/usergrid/persistence/Schema.java | 5 +-
.../persistence/cassandra/ApplicationCF.java | 37 -
.../cassandra/CassandraPersistenceUtils.java | 122 +-
.../persistence/cassandra/CassandraService.java | 304 +-
.../persistence/cassandra/CursorCache.java | 137 -
.../cassandra/EntityManagerFactoryImpl.java | 485 ---
.../cassandra/EntityManagerImpl.java | 2946 ------------------
.../cassandra/EntityValueSerializer.java | 53 -
.../persistence/cassandra/GeoIndexManager.java | 330 --
.../persistence/cassandra/IndexUpdate.java | 451 ---
.../persistence/cassandra/QueryProcessor.java | 60 -
.../cassandra/QueryProcessorImpl.java | 727 -----
.../cassandra/RelationManagerImpl.java | 2298 --------------
.../persistence/cassandra/SetupImpl.java | 170 -
.../cassandra/SimpleIndexBucketLocatorImpl.java | 122 -
.../cassandra/index/ConnectedIndexScanner.java | 280 --
.../cassandra/index/IndexBucketScanner.java | 238 --
.../index/IndexMultiBucketSetLoader.java | 139 -
.../cassandra/index/IndexScanner.java | 40 -
.../cassandra/index/NoOpIndexScanner.java | 95 -
.../DuplicateUniquePropertyExistsException.java | 2 +-
.../persistence/geo/CollectionGeoSearch.java | 68 -
.../persistence/geo/ConnectionGeoSearch.java | 67 -
.../persistence/geo/EntityLocationRef.java | 227 --
.../EntityLocationRefDistanceComparator.java | 78 -
.../persistence/geo/GeoIndexSearcher.java | 370 ---
.../persistence/geo/GeocellManager.java | 195 --
.../usergrid/persistence/geo/GeocellUtils.java | 543 ----
.../geo/comparator/DoubleTupleComparator.java | 39 -
.../persistence/geo/model/BoundingBox.java | 74 -
.../persistence/geo/model/CostFunction.java | 36 -
.../geo/model/DefaultCostFunction.java | 37 -
.../usergrid/persistence/geo/model/Point.java | 61 -
.../usergrid/persistence/geo/model/Tuple.java | 40 -
.../usergrid/persistence/query/ir/AllNode.java | 82 -
.../usergrid/persistence/query/ir/AndNode.java | 44 -
.../persistence/query/ir/BooleanNode.java | 65 -
.../query/ir/EmailIdentifierNode.java | 58 -
.../query/ir/NameIdentifierNode.java | 56 -
.../persistence/query/ir/NodeVisitor.java | 79 -
.../usergrid/persistence/query/ir/NotNode.java | 75 -
.../usergrid/persistence/query/ir/OrNode.java | 53 -
.../persistence/query/ir/OrderByNode.java | 105 -
.../persistence/query/ir/QueryNode.java | 41 -
.../persistence/query/ir/QuerySlice.java | 345 --
.../persistence/query/ir/SearchVisitor.java | 270 --
.../persistence/query/ir/SliceNode.java | 180 --
.../query/ir/UuidIdentifierNode.java | 60 -
.../persistence/query/ir/WithinNode.java | 109 -
.../query/ir/result/AbstractScanColumn.java | 83 -
.../result/CollectionResultsLoaderFactory.java | 41 -
.../ir/result/ConnectionIndexSliceParser.java | 87 -
.../query/ir/result/ConnectionRefLoader.java | 81 -
.../result/ConnectionResultsLoaderFactory.java | 50 -
.../ir/result/ConnectionTypesIterator.java | 190 --
.../query/ir/result/EmptyIterator.java | 63 -
.../query/ir/result/EntityRefLoader.java | 51 -
.../query/ir/result/EntityResultsLoader.java | 47 -
.../query/ir/result/GeoIterator.java | 351 ---
.../persistence/query/ir/result/IDLoader.java | 40 -
.../query/ir/result/IntersectionIterator.java | 170 -
.../query/ir/result/MergeIterator.java | 150 -
.../query/ir/result/MultiIterator.java | 55 -
.../query/ir/result/OrderByIterator.java | 250 --
.../query/ir/result/ResultIterator.java | 44 -
.../query/ir/result/ResultsLoader.java | 30 -
.../query/ir/result/ResultsLoaderFactory.java | 37 -
.../persistence/query/ir/result/ScanColumn.java | 32 -
.../query/ir/result/ScanColumnTransformer.java | 39 -
.../ir/result/SecondaryIndexSliceParser.java | 61 -
.../query/ir/result/SliceIterator.java | 242 --
.../query/ir/result/SliceParser.java | 32 -
.../query/ir/result/StaticIdIterator.java | 82 -
.../query/ir/result/SubtractionIterator.java | 113 -
.../query/ir/result/UUIDIndexSliceParser.java | 47 -
.../query/ir/result/UnionIterator.java | 264 --
.../main/resources/usergrid-core-context.xml | 32 +-
.../java/org/apache/usergrid/Application.java | 2 +-
.../org/apache/usergrid/CoreApplication.java | 37 +-
.../java/org/apache/usergrid/CoreITSetup.java | 10 +-
.../org/apache/usergrid/CoreITSetupImpl.java | 69 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 +
.../usergrid/batch/job/SchedulerRuntime8IT.java | 8 +-
.../corepersistence/StaleIndexCleanupTest.java | 111 +-
.../corepersistence/TestIndexModule.java | 43 +
.../index/AsyncIndexServiceTest.java | 203 ++
.../index/InMemoryAsycIndexServiceTest.java | 64 +
.../corepersistence/index/IndexServiceTest.java | 327 ++
.../corepersistence/index/PublishRxTest.java | 95 +
.../index/SQSAsyncIndexServiceTest.java | 108 +
.../migration/EntityDataMigrationIT.java | 262 --
.../migration/EntityTypeMappingMigrationIT.java | 183 +-
.../migration/GraphShardVersionMigrationIT.java | 226 --
.../MigrationModuleVersionPluginTest.java | 259 ++
.../migration/MigrationTestRule.java | 99 -
.../migration/TestProgressObserver.java | 71 -
.../rx/AllEntitiesInSystemObservableIT.java | 71 +-
.../rx/ApplicationObservableTestIT.java | 17 +-
.../rx/EdgesFromSourceObservableIT.java | 19 +-
.../rx/EdgesToTargetObservableIT.java | 59 +-
.../rx/TargetIdObservableTestIT.java | 7 +-
.../usergrid/persistence/CollectionIT.java | 379 ++-
.../apache/usergrid/persistence/CounterIT.java | 11 +-
.../usergrid/persistence/CountingMutatorIT.java | 6 +-
.../persistence/EntityConnectionsIT.java | 16 +-
.../usergrid/persistence/EntityManagerIT.java | 62 +-
.../EntityPropertyComparatorTest.java | 222 --
.../org/apache/usergrid/persistence/GeoIT.java | 122 +-
.../persistence/GeoQueryBooleanTest.java | 24 +-
.../apache/usergrid/persistence/IndexIT.java | 64 +-
.../usergrid/persistence/LargeEntityIT.java | 3 +-
.../usergrid/persistence/PathQueryIT.java | 27 +-
.../PerformanceEntityRebuildIndexTest.java | 60 +-
.../usergrid/persistence/PermissionsIT.java | 6 +-
.../apache/usergrid/persistence/QueryTest.java | 328 --
.../cassandra/EntityManagerFactoryImplIT.java | 126 +-
.../cassandra/QueryProcessorTest.java | 822 -----
.../SimpleIndexShardLocatorImplTest.java | 177 --
...EntityLocationRefDistanceComparatorTest.java | 104 -
.../persistence/query/CollectionIoHelper.java | 4 +-
.../persistence/query/ConnectionHelper.java | 4 +-
.../query/IntersectionTransitivePagingIT.java | 6 +-
.../query/IntersectionUnionPagingIT.java | 6 +-
.../usergrid/persistence/query/IoHelper.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 157 +-
.../persistence/query/NotSubPropertyIT.java | 4 +-
.../query/OrderByDiscardPagesPagingIT.java | 164 -
.../query/ir/result/AbstractScanColumnTest.java | 101 -
.../query/ir/result/InOrderIterator.java | 132 -
.../ir/result/IntersectionIteratorTest.java | 308 --
.../query/ir/result/IteratorHelper.java | 34 -
.../ir/result/SubtractionIteratorTest.java | 182 --
.../query/ir/result/UnionIteratorTest.java | 468 ---
.../apache/usergrid/utils/EdgeTestUtils.java | 50 +
stack/core/src/test/resources/log4j.properties | 4 +
.../resources/usergrid-custom-test.properties | 4 +-
.../resources/usergrid-properties-context.xml | 40 -
.../test/resources/usergrid-test-context.xml | 1 +
stack/corepersistence/collection/pom.xml | 14 +-
.../persistence/collection/CollectionScope.java | 45 -
.../collection/EntityCollectionManager.java | 55 +-
.../EntityCollectionManagerFactory.java | 36 +-
.../collection/EntityCollectionManagerSync.java | 49 -
.../persistence/collection/FieldSet.java | 2 +-
.../persistence/collection/MvccEntity.java | 5 +-
.../persistence/collection/MvccLogEntry.java | 1 +
.../cache/CachedEntityCollectionManager.java | 21 +-
.../collection/event/EntityDeleted.java | 8 +-
.../collection/event/EntityVersionCreated.java | 7 +-
.../collection/event/EntityVersionDeleted.java | 11 +-
.../exception/CollectionRuntimeException.java | 30 +-
.../exception/EntityTooLargeException.java | 6 +-
.../exception/WriteCommitException.java | 11 +-
.../WriteOptimisticVerifyException.java | 11 +-
.../exception/WriteStartException.java | 11 +-
.../exception/WriteUniqueVerifyException.java | 6 +-
.../collection/guice/CollectionModule.java | 48 +-
.../guice/CollectionTaskExecutor.java | 4 +-
.../persistence/collection/guice/Write.java | 17 -
.../collection/guice/WriteUpdate.java | 17 -
.../collection/impl/CollectionScopeImpl.java | 102 -
.../EntityCollectionManagerFactoryImpl.java | 93 +-
.../impl/EntityCollectionManagerImpl.java | 197 +-
.../impl/EntityCollectionManagerSyncImpl.java | 71 -
.../collection/impl/EntityDeletedTask.java | 58 +-
.../impl/EntityVersionCleanupTask.java | 197 +-
.../impl/EntityVersionCreatedTask.java | 39 +-
.../impl/EntityVersionTaskFactory.java | 17 +-
.../mvcc/MvccEntitySerializationStrategy.java | 102 -
.../mvcc/MvccLogEntrySerializationStrategy.java | 80 -
.../collection/mvcc/changelog/ChangeLog.java | 2 -
.../mvcc/entity/MvccValidationUtils.java | 16 -
.../mvcc/entity/impl/MvccEntityDeleteEvent.java | 8 +-
.../mvcc/entity/impl/MvccEntityEvent.java | 11 +-
.../mvcc/entity/impl/MvccEntityWriteEvent.java | 6 +-
.../mvcc/stage/CollectionIoEvent.java | 9 +-
.../mvcc/stage/EntityUpdateEvent.java | 34 -
.../mvcc/stage/delete/MarkCommit.java | 15 +-
.../collection/mvcc/stage/delete/MarkStart.java | 20 +-
.../mvcc/stage/write/RollbackAction.java | 14 +-
.../mvcc/stage/write/WriteCommit.java | 32 +-
.../mvcc/stage/write/WriteOptimisticVerify.java | 12 +-
.../collection/mvcc/stage/write/WriteStart.java | 22 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 45 +-
.../collection/serialization/EntityRepair.java | 38 -
.../MvccEntitySerializationStrategy.java | 117 +
.../MvccLogEntrySerializationStrategy.java | 82 +
.../serialization/OptimisticUpdate.java | 23 -
.../collection/serialization/UniqueValue.java | 1 -
.../UniqueValueSerializationStrategy.java | 65 +-
.../serialization/UniqueValueSet.java | 1 +
.../impl/CollectionDataVersions.java | 54 +
.../serialization/impl/EntityRepairImpl.java | 149 -
.../impl/EntityVersionSerializer.java | 11 +-
.../serialization/impl/FieldSerializer.java | 118 -
.../serialization/impl/LogEntryIterator.java | 8 +-
.../serialization/impl/MutableFieldSet.java | 6 +-
.../MvccEntitySerializationStrategyImpl.java | 177 +-
...vccEntitySerializationStrategyProxyImpl.java | 163 +-
.../MvccEntitySerializationStrategyV1Impl.java | 9 +-
.../MvccEntitySerializationStrategyV2Impl.java | 8 +-
.../MvccEntitySerializationStrategyV3Impl.java | 574 ++++
.../MvccLogEntrySerializationProxyImpl.java | 149 +
.../MvccLogEntrySerializationStrategyImpl.java | 92 +-
...MvccLogEntrySerializationStrategyV1Impl.java | 118 +
...MvccLogEntrySerializationStrategyV2Impl.java | 106 +
.../serialization/impl/SerializationModule.java | 170 +-
.../serialization/impl/TypeField.java | 43 +
.../serialization/impl/UniqueFieldEntry.java | 60 +
.../impl/UniqueFieldEntrySerializer.java | 138 +
.../impl/UniqueFieldRowKeySerializer.java | 122 +
.../impl/UniqueTypeFieldRowKeySerializer.java | 60 +
.../serialization/impl/UniqueValueImpl.java | 5 +-
.../UniqueValueSerializationStrategyImpl.java | 322 +-
...iqueValueSerializationStrategyProxyImpl.java | 179 ++
.../UniqueValueSerializationStrategyV1Impl.java | 159 +
.../UniqueValueSerializationStrategyV2Impl.java | 139 +
.../impl/migration/CollectionMigration.java | 53 +
.../migration/CollectionMigrationPlugin.java | 68 +
.../impl/migration/EntityIdScope.java | 79 +
.../migration/MvccEntityDataMigrationImpl.java | 280 ++
.../serialization/impl/util/Inflector.java | 370 +++
.../impl/util/LegacyScopeUtils.java | 51 +
.../collection/util/EntityUtils.java | 71 -
.../collection/ApplicationContextImplTest.java | 33 +
.../collection/CollectionContextImplTest.java | 53 -
.../EntityCollectionManagerFactoryTest.java | 9 +-
.../collection/EntityCollectionManagerIT.java | 271 +-
.../EntityCollectionManagerStressTest.java | 14 +-
.../EntityCollectionManagerSyncIT.java | 193 --
.../collection/guice/TestCollectionModule.java | 22 +-
.../impl/EntityVersionCleanupTaskTest.java | 683 ++--
.../impl/EntityVersionCreatedTaskTest.java | 29 +-
.../mvcc/stage/AbstractEntityStageTest.java | 6 +-
.../mvcc/stage/AbstractIdStageTest.java | 4 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 6 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 14 +-
.../mvcc/stage/delete/MarkStartTest.java | 6 +-
.../mvcc/stage/write/FieldSerializerTest.java | 52 -
.../write/UniqueFieldRowKeySerializerTest.java | 52 +
...niqueValueSerializationStrategyImplTest.java | 185 --
.../mvcc/stage/write/WriteCommitTest.java | 12 +-
.../stage/write/WriteOptimisticVerifyTest.java | 35 +-
.../mvcc/stage/write/WriteStartTest.java | 26 +-
.../mvcc/stage/write/WriteUniqueVerifyIT.java | 34 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 13 +-
.../persistence/collection/rx/ParallelTest.java | 10 +-
.../serialization/EntityRepairImplTest.java | 147 -
.../impl/LogEntryIteratorTest.java | 17 +-
...MvccEntitySerializationStrategyImplTest.java | 165 +-
...cEntitySerializationStrategyProxyV1Test.java | 85 -
...ntitySerializationStrategyProxyV1_3Test.java | 81 +
...cEntitySerializationStrategyProxyV2Test.java | 83 -
...ntitySerializationStrategyProxyV2_3Test.java | 82 +
...ccEntitySerializationStrategyV1ImplTest.java | 78 +-
...ccEntitySerializationStrategyV2ImplTest.java | 73 +-
.../MvccEntitySerializationStrategyV2Test.java | 27 +-
...ccEntitySerializationStrategyV3ImplTest.java | 70 +
.../impl/MvccLESSTransientTest.java | 27 +-
...ccLogEntrySerializationStrategyImplTest.java | 107 +-
...EntrySerializationStrategyProxyImplTest.java | 90 +
...LogEntrySerializationStrategyV1ImplTest.java | 55 +
...LogEntrySerializationStrategyV2ImplTest.java | 53 +
.../impl/ScopedRowKeySerializerTest.java | 2 +-
.../impl/SerializationComparison.java | 4 +-
.../impl/SettingsValidationTest.java | 1 -
.../impl/UniqueFieldEntrySerializerTest.java | 108 +
...niqueValueSerializationStrategyImplTest.java | 344 ++
...ValueSerializationStrategyProxyImplTest.java | 100 +
...queValueSerializationStrategyV1ImplTest.java | 57 +
...queValueSerializationStrategyV2ImplTest.java | 58 +
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 214 ++
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 99 +
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 99 +
.../collection/util/EntityHelper.java | 1 -
.../collection/util/InvalidEntityGenerator.java | 1 +
.../collection/util/LogEntryMock.java | 47 +-
.../collection/util/UniqueValueEntryMock.java | 150 +
.../collection/util/VersionGenerator.java | 55 +
stack/corepersistence/common/pom.xml | 21 +-
.../usergrid/persistence/core/CPManager.java | 28 +
.../persistence/core/astyanax/CassandraFig.java | 3 -
.../astyanax/DynamicCompositeParserImpl.java | 81 +
.../core/astyanax/FieldBufferBuilder.java | 15 +
.../core/astyanax/FieldBufferParser.java | 13 +
.../astyanax/MultiKeyColumnNameIterator.java | 4 +-
.../core/astyanax/MultiRowColumnIterator.java | 12 +-
.../persistence/core/guice/CommonModule.java | 27 +-
.../persistence/core/guice/CurrentImpl.java | 42 -
.../persistence/core/guice/PreviousImpl.java | 42 -
.../persistence/core/guice/ProxyImpl.java | 42 -
.../core/hystrix/HystrixCassandra.java | 94 -
.../core/metrics/MetricsFactory.java | 2 +-
.../core/metrics/MetricsFactoryImpl.java | 44 +-
.../core/metrics/ObservableTimer.java | 72 +
.../migration/data/AbstractMigrationPlugin.java | 128 +
.../core/migration/data/DataMigration.java | 97 +-
.../migration/data/DataMigrationManager.java | 18 +-
.../data/DataMigrationManagerImpl.java | 237 +-
.../migration/data/MigrationDataProvider.java | 43 +
.../core/migration/data/MigrationInfoCache.java | 52 +
.../migration/data/MigrationInfoCacheImpl.java | 84 +
.../data/MigrationInfoSerialization.java | 28 +-
.../data/MigrationInfoSerializationImpl.java | 67 +-
.../core/migration/data/MigrationPlugin.java | 56 +
.../migration/data/MigrationRelationship.java | 117 +
.../core/migration/data/PluginPhase.java | 42 +
.../core/migration/data/ProgressObserver.java | 63 +
.../core/migration/data/VersionedData.java | 38 +
.../migration/data/VersionedMigrationSet.java | 164 +
.../core/migration/schema/Migration.java | 2 +-
.../persistence/core/rx/ObservableIterator.java | 2 +
.../persistence/core/rx/RxSchedulerFig.java | 60 +
.../persistence/core/rx/RxTaskScheduler.java | 40 +
.../core/rx/RxTaskSchedulerImpl.java | 131 +
.../core/scope/ApplicationScope.java | 9 +-
.../core/scope/ApplicationScopeImpl.java | 24 +-
.../persistence/core/task/TaskExecutor.java | 4 +-
.../persistence/core/util/StringUtils.java | 34 +
.../MultiKeyColumnNameIteratorTest.java | 187 +-
.../astyanax/MultiRowColumnIteratorTest.java | 50 +-
.../persistence/core/aws/NoAWSCredsRule.java | 98 +
.../core/guice/DataMigrationResetRule.java | 88 +
.../core/guice/MaxMigrationModule.java | 39 -
.../core/guice/MaxMigrationVersion.java | 40 -
.../core/guice/MigrationManagerRule.java | 7 +
.../core/guice/TestCommonModule.java | 1 +
.../persistence/core/guice/TestModule.java | 5 +-
.../data/DataMigrationManagerImplTest.java | 350 ++-
.../data/MigrationInfoSerializationTest.java | 32 +-
.../data/TestMigrationDataProvider.java | 61 +
.../migration/data/TestProgressObserver.java | 89 +
.../data/VersionedMigrationSetTest.java | 194 ++
.../persistence/core/util/IdGenerator.java | 51 +
.../persistence/graph/GraphManager.java | 3 +-
.../persistence/graph/GraphManagerFactory.java | 2 +
.../persistence/graph/guice/GraphModule.java | 106 +-
.../graph/impl/GraphManagerImpl.java | 41 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 21 +-
.../impl/stage/NodeDeleteListenerImpl.java | 21 +-
.../EdgeMetadataSerialization.java | 3 +-
.../graph/serialization/EdgesObservable.java | 71 +
.../graph/serialization/TargetIdObservable.java | 38 +
.../EdgeMetadataSerializationProxyImpl.java | 161 +-
.../impl/EdgeMetadataSerializationV1Impl.java | 6 +
.../impl/EdgeMetadataSerializationV2Impl.java | 6 +
.../serialization/impl/EdgesObservableImpl.java | 129 +
.../serialization/impl/GraphDataVersions.java | 43 +
.../impl/GraphManagerFactoryImpl.java | 95 +
.../impl/NodeSerializationImpl.java | 36 +-
.../impl/TargetIdObservableImpl.java | 72 +
.../impl/migration/EdgeDataMigrationImpl.java | 136 +
.../impl/migration/GraphMigration.java | 53 +
.../impl/migration/GraphMigrationPlugin.java | 69 +
.../serialization/impl/migration/GraphNode.java | 39 +
.../shard/count/NodeShardApproximationImpl.java | 13 +-
.../NodeShardCounterSerializationImpl.java | 25 +-
.../shard/impl/NodeShardAllocationImpl.java | 16 +-
.../shard/impl/ShardGroupCompactionImpl.java | 64 +-
.../persistence/graph/GraphManagerIT.java | 35 +-
.../persistence/graph/GraphManagerLoadTest.java | 13 +-
.../graph/GraphManagerShardConsistencyIT.java | 12 +-
.../graph/GraphManagerShardingIT.java | 13 +-
.../graph/GraphManagerStressTest.java | 11 +-
.../usergrid/persistence/graph/SimpleTest.java | 12 +-
.../graph/guice/TestGraphModule.java | 27 +-
.../graph/impl/EdgeDeleteListenerTest.java | 12 +-
.../graph/impl/NodeDeleteListenerTest.java | 10 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 7 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 30 +-
.../EdgeMetaDataSerializationProxyV1Test.java | 15 +-
.../EdgeMetaDataSerializationProxyV2Test.java | 23 +-
.../EdgeMetaDataSerializationV1Test.java | 7 +-
.../EdgeMetaDataSerializationV2Test.java | 7 +-
.../EdgeMetadataSerializationTest.java | 39 +-
.../EdgeSerializationChopTest.java | 7 +-
.../serialization/EdgeSerializationTest.java | 21 +-
.../serialization/NodeSerializationTest.java | 15 +-
.../migration/EdgeDataMigrationImplTest.java | 177 ++
.../impl/shard/EdgeShardSerializationTest.java | 7 +-
.../impl/shard/NodeShardAllocationTest.java | 27 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../impl/shard/ShardGroupCompactionTest.java | 7 +-
.../shard/count/NodeShardApproximationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 6 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 15 +-
...rceDirectedEdgeDescendingComparatorTest.java | 23 +-
...getDirectedEdgeDescendingComparatorTest.java | 23 +-
.../graph/test/util/EdgeTestUtils.java | 29 +-
.../usergrid/persistence/map/MapManager.java | 25 +-
.../persistence/map/guice/MapModule.java | 8 +-
.../persistence/map/guice/TestMapModule.java | 3 +-
stack/corepersistence/model/pom.xml | 1 -
.../persistence/model/entity/Entity.java | 23 +-
.../persistence/model/entity/EntityMap.java | 80 +
.../model/entity/EntityToMapConverter.java | 160 +
.../model/entity/MapToEntityConverter.java | 203 ++
.../persistence/model/entity/SimpleId.java | 36 +-
.../persistence/model/field/AbstractField.java | 8 +
.../persistence/model/field/ListField.java | 8 +-
.../persistence/model/field/StringField.java | 20 +
.../persistence/model/util/EntityUtils.java | 72 +
stack/corepersistence/pom.xml | 36 +-
stack/corepersistence/queryindex/pom.xml | 63 +-
.../index/query/tree/CpQueryFilter.g | 20 +-
.../persistence/index/AliasedEntityIndex.java | 13 +-
.../index/ApplicationEntityIndex.java | 55 +
.../persistence/index/CandidateResult.java | 84 +
.../persistence/index/CandidateResults.java | 106 +
.../usergrid/persistence/index/EntityIndex.java | 75 +-
.../persistence/index/EntityIndexBatch.java | 25 +-
.../persistence/index/EntityIndexFactory.java | 11 +-
.../persistence/index/IndexBatchBuffer.java | 36 -
.../persistence/index/IndexBufferConsumer.java | 37 -
.../persistence/index/IndexBufferProducer.java | 37 -
.../usergrid/persistence/index/IndexEdge.java | 36 +
.../usergrid/persistence/index/IndexFig.java | 128 +-
.../persistence/index/IndexIdentifier.java | 92 -
.../index/IndexOperationMessage.java | 138 -
.../persistence/index/IndexRefreshCommand.java | 50 +
.../usergrid/persistence/index/IndexScope.java | 38 -
.../usergrid/persistence/index/SearchEdge.java | 61 +
.../usergrid/persistence/index/SearchType.java | 51 +
.../usergrid/persistence/index/SearchTypes.java | 46 +-
.../persistence/index/SelectFieldMapping.java | 58 +
.../persistence/index/guice/IndexModule.java | 44 +-
.../persistence/index/guice/QueueProvider.java | 116 -
.../persistence/index/impl/BatchOperation.java | 41 +
.../persistence/index/impl/BatchRequest.java | 41 -
.../persistence/index/impl/BufferQueue.java | 68 -
.../index/impl/BufferQueueInMemoryImpl.java | 116 -
.../index/impl/BufferQueueSQSImpl.java | 307 --
.../index/impl/DeIndexOperation.java | 104 +
.../persistence/index/impl/DeIndexRequest.java | 115 -
.../persistence/index/impl/EntityField.java | 148 +
.../index/impl/EntityMappingParser.java | 247 ++
.../index/impl/EntityToMapConverter.java | 108 +
.../impl/EsApplicationEntityIndexImpl.java | 274 ++
.../index/impl/EsEntityIndexBatchImpl.java | 283 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 44 +-
.../index/impl/EsEntityIndexImpl.java | 590 +---
.../index/impl/EsIndexBufferConsumerImpl.java | 348 +--
.../index/impl/EsIndexBufferProducerImpl.java | 65 -
.../persistence/index/impl/EsIndexCache.java | 142 -
.../index/impl/EsIndexCacheImpl.java | 141 +
.../persistence/index/impl/EsProvider.java | 129 +-
.../persistence/index/impl/EsQueryVistor.java | 589 ++--
.../persistence/index/impl/FieldParser.java | 45 +
.../index/impl/FlushBufferQueue.java | 23 +
.../persistence/index/impl/GeoSortFields.java | 95 +
.../persistence/index/impl/IndexAlias.java | 42 +
.../index/impl/IndexBufferConsumer.java | 38 +
.../persistence/index/impl/IndexCache.java | 43 +
.../persistence/index/impl/IndexEdgeImpl.java | 84 +
.../persistence/index/impl/IndexIdentifier.java | 46 +
.../index/impl/IndexIdentifierImpl.java | 67 +
.../persistence/index/impl/IndexOperation.java | 107 +
.../index/impl/IndexOperationMessage.java | 126 +
.../index/impl/IndexRefreshCommandImpl.java | 170 +
.../persistence/index/impl/IndexRequest.java | 125 -
.../persistence/index/impl/IndexScopeImpl.java | 88 -
.../persistence/index/impl/IndexingUtils.java | 267 +-
.../index/impl/NoOpFilterBuilder.java | 45 +
.../index/impl/NoOpQueryBuilder.java | 45 +
.../persistence/index/impl/SearchEdgeImpl.java | 98 +
.../impl/SearchRequestBuilderStrategy.java | 290 ++
.../persistence/index/impl/SortBuilder.java | 44 +
.../index/migration/EsIndexMigrationPlugin.java | 46 +
.../index/migration/IndexDataVersions.java | 39 +
.../index/migration/IndexMigration.java | 35 +
.../index/migration/LegacyIndexIdentifier.java | 78 +
.../index/query/CandidateResult.java | 74 -
.../index/query/CandidateResults.java | 90 -
.../persistence/index/query/EntityResults.java | 108 -
.../persistence/index/query/ParsedQuery.java | 205 ++
.../index/query/ParsedQueryBuilder.java | 108 +
.../usergrid/persistence/index/query/Query.java | 1436 ---------
.../persistence/index/query/Results.java | 148 -
.../persistence/index/query/SortPredicate.java | 144 +
.../index/query/tree/QueryVisitor.java | 55 +-
.../index/utils/IndexValidationUtils.java | 24 +-
.../persistence/index/utils/ListUtils.java | 1 +
.../persistence/index/utils/StringUtils.java | 62 -
.../persistence/index/usergrid-mappings.json | 93 +
.../persistence/index/guice/IndexTestFig.java | 62 +
.../index/guice/TestIndexModule.java | 42 +-
.../index/impl/BufferQueueSQSImplTest.java | 169 -
.../index/impl/CorePerformanceIT.java | 339 --
.../index/impl/ElasticSearchRule.java | 58 +
.../impl/EntityConnectionIndexImplTest.java | 306 --
.../index/impl/EntityIndexMapUtils.java | 52 +-
.../persistence/index/impl/EntityIndexTest.java | 1073 +++++--
.../index/impl/EntityToMapConverterTest.java | 653 ++++
.../persistence/index/impl/GeoPagingTest.java | 163 +
.../index/impl/IndexLoadTestsIT.java | 393 +++
.../persistence/query/tree/GrammarTreeTest.java | 157 +-
.../src/test/resources/dynamic-test.properties | 2 +-
.../src/test/resources/log4j.properties | 10 +-
.../src/test/resources/usergrid-UNIT.properties | 3 +
.../usergrid/persistence/queue/QueueFig.java | 8 +
.../persistence/queue/NoAWSCredsRule.java | 98 -
.../persistence/queue/QueueManagerTest.java | 3 +-
.../queue/guice/TestQueueModule.java | 3 +-
stack/java-sdk-old/pom.xml | 102 -
.../usergrid/mongo/protocol/OpDelete.java | 4 +-
.../apache/usergrid/mongo/protocol/OpQuery.java | 6 +-
.../usergrid/mongo/protocol/OpUpdate.java | 4 +-
.../apache/usergrid/mongo/BasicMongoTest.java | 8 +-
.../apache/usergrid/mongo/MongoQueryTest.java | 18 +-
stack/pom.xml | 28 +-
stack/rest/pom.xml | 17 +-
.../usergrid/rest/AbstractContextResource.java | 102 +-
.../org/apache/usergrid/rest/ApiResponse.java | 50 +-
.../org/apache/usergrid/rest/IndexResource.java | 9 +-
.../apache/usergrid/rest/MigrateResource.java | 122 +-
.../org/apache/usergrid/rest/RootResource.java | 19 +-
.../rest/applications/ApplicationResource.java | 38 +-
.../rest/applications/ServiceResource.java | 43 +-
.../notifiers/NotifiersResource.java | 11 +-
.../rest/applications/users/UsersResource.java | 4 +-
.../exceptions/AbstractExceptionMapper.java | 11 +-
.../exceptions/ConflictExceptionMapper.java | 35 +
.../exceptions/NotFoundExceptionMapper.java | 35 +
.../rest/exceptions/UncaughtException.java | 36 +
.../rest/management/ManagementResource.java | 59 +-
.../organizations/OrganizationResource.java | 28 +-
.../organizations/OrganizationsResource.java | 23 +-
.../applications/ApplicationResource.java | 61 +-
.../applications/ApplicationsResource.java | 34 +-
.../imports/FileIncludesResource.java | 9 -
.../applications/imports/ImportsResource.java | 83 +-
.../organizations/OrganizationResource.java | 32 +-
.../OAuth2AccessTokenSecurityFilter.java | 2 +-
.../rest/test/RefreshIndexResource.java | 25 +-
.../java/org/apache/usergrid/rest/BasicIT.java | 302 +-
.../apache/usergrid/rest/IndexResourceIT.java | 1 +
.../apache/usergrid/rest/NotificationsIT.java | 93 +-
.../apache/usergrid/rest/SystemResourceIT.java | 10 +-
.../rest/applications/ApplicationCreateIT.java | 135 +
.../rest/applications/ApplicationDeleteIT.java | 389 +++
.../applications/ApplicationDeleteTest.java | 54 -
.../applications/ApplicationResourceIT.java | 100 +-
.../collection/CollectionsResourceIT.java | 19 +-
.../collection/paging/PagingResourceIT.java | 17 +-
.../collection/users/OwnershipResourceIT.java | 4 +-
.../collection/users/PermissionsResourceIT.java | 15 -
.../collection/users/RetrieveUsersTest.java | 4 +-
.../collection/users/UserResourceIT.java | 16 +-
.../events/ApplicationRequestCounterIT.java | 2 +-
.../applications/queries/BasicGeoTests.java | 21 +-
.../applications/queries/GeoPagingTest.java | 58 +-
.../applications/queries/MatrixQueryTests.java | 298 +-
.../rest/applications/queries/OrderByTest.java | 2 +-
.../queues/AbstractQueueResourceIT.java | 484 ---
.../queues/QueueResourceLong1IT.java | 149 -
.../queues/QueueResourceLong2IT.java | 99 -
.../queues/QueueResourceLong3IT.java | 201 --
.../queues/QueueResourceShortIT.java | 432 ---
.../rest/filters/ContentTypeResourceIT.java | 55 +-
.../usergrid/rest/management/AccessTokenIT.java | 418 +--
.../usergrid/rest/management/AdminUsersIT.java | 64 +-
.../rest/management/ExportResourceIT.java | 674 ++--
.../rest/management/ImportResourceIT.java | 179 +-
.../rest/management/ManagementResourceIT.java | 151 +-
.../rest/management/OrganizationsIT.java | 82 +-
.../organizations/AdminEmailEncodingIT.java | 3 +-
.../test/resource2point0/AbstractRestIT.java | 4 +-
.../rest/test/resource2point0/ClientSetup.java | 30 +-
.../rest/test/resource2point0/README.md | 111 +
.../rest/test/resource2point0/RestClient.java | 1 +
.../endpoints/ApplicationsResource.java | 34 +-
.../endpoints/CollectionEndpoint.java | 13 +
.../endpoints/NamedResource.java | 158 +-
.../endpoints/OrganizationResource.java | 6 +-
.../resource2point0/endpoints/RootResource.java | 1 +
.../endpoints/SetupResource.java | 6 +
.../endpoints/mgmt/ApplicationResource.java | 54 +-
.../endpoints/mgmt/ApplicationsResource.java | 58 +
.../endpoints/mgmt/ManagementResource.java | 4 +
.../endpoints/mgmt/ManagementResponse.java | 72 +
.../endpoints/mgmt/MeResource.java | 32 +
.../mgmt/OrganizationApplicationResponse.java | 60 +
.../endpoints/mgmt/OrganizationResource.java | 5 +-
.../endpoints/mgmt/PasswordResource.java | 16 -
.../endpoints/mgmt/RevokeTokenResource.java | 32 +
.../endpoints/mgmt/RevokeTokensResource.java | 33 +
.../endpoints/mgmt/TokenResource.java | 25 +-
.../endpoints/mgmt/UserResource.java | 8 +
.../endpoints/mgmt/UsersResource.java | 15 -
stack/rest/src/test/resources/log4j.properties | 10 +-
.../resources/testImportCorrect.testCol.1.json | 18 -
...testImportInvalidJson.testApplication.3.json | 153 -
.../testimport-bad-json-testapp.3.json | 153 +
.../src/test/resources/testimport-bad-json.json | 67 +
.../resources/testimport-correct-testcol.1.json | 18 +
.../test/resources/testimport-test-app.2.json | 56 +
.../resources/usergrid-custom-test.properties | 2 +
stack/services/pom.xml | 51 +-
.../usergrid/management/ApplicationInfo.java | 4 +
.../usergrid/management/ManagementService.java | 10 +-
.../cassandra/ApplicationCreatorImpl.java | 12 +-
.../cassandra/ManagementServiceImpl.java | 236 +-
.../management/export/ExportServiceImpl.java | 4 +-
.../management/importer/FileImportJob.java | 8 +-
.../management/importer/FileImportTracker.java | 6 +-
.../usergrid/management/importer/ImportJob.java | 4 +-
.../management/importer/ImportServiceImpl.java | 88 +-
.../usergrid/security/AuthPrincipalType.java | 3 +-
.../security/providers/AbstractProvider.java | 1 +
.../security/providers/FacebookProvider.java | 6 +-
.../security/providers/FoursquareProvider.java | 6 +-
.../shiro/PrincipalCredentialsToken.java | 20 +-
.../apache/usergrid/security/shiro/Realm.java | 2 +-
.../services/AbstractCollectionService.java | 17 +-
.../services/AbstractConnectionsService.java | 4 +-
.../AbstractPathBasedColllectionService.java | 2 +-
.../usergrid/services/AbstractService.java | 2 +-
.../usergrid/services/ServiceContext.java | 2 +-
.../services/ServiceManagerFactory.java | 4 -
.../usergrid/services/ServiceParameter.java | 3 +-
.../usergrid/services/ServiceRequest.java | 2 +-
.../usergrid/services/ServiceResults.java | 38 +-
.../applications/ApplicationsService.java | 2 +-
.../services/devices/DevicesService.java | 2 +-
.../usergrid/services/groups/GroupsService.java | 2 +-
.../groups/activities/ActivitiesService.java | 2 +-
.../users/activities/ActivitiesService.java | 4 +-
.../notifications/InactiveDeviceManager.java | 8 +-
.../notifications/NotificationsService.java | 4 +-
.../apns/FailedConnectionListener.java | 31 +-
.../impl/ApplicationQueueManagerImpl.java | 197 +-
.../services/notifications/wns/WNSAdapter.java | 60 +-
.../usergrid/services/roles/RolesService.java | 10 +-
.../usergrid/services/users/UsersService.java | 2 +-
.../users/activities/ActivitiesService.java | 4 +-
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../org/apache/usergrid/ServiceITSetup.java | 5 +
.../org/apache/usergrid/ServiceITSetupImpl.java | 43 +-
.../migration/AppInfoMigrationPluginTest.java | 231 ++
.../apache/usergrid/management/EmailFlowIT.java | 9 +-
.../usergrid/management/OrganizationIT.java | 16 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ApplicationCreatorIT.java | 14 +-
.../cassandra/ManagementServiceIT.java | 35 +-
.../management/export/ExportServiceIT.java | 20 +-
.../importer/FileImportTrackerTest.java | 12 +-
.../management/importer/ImportCollectionIT.java | 20 +-
.../importer/ImportConnectionsTest.java | 9 +-
.../management/importer/ImportServiceIT.java | 2 +-
.../security/tokens/TokenServiceIT.java | 2 -
.../usergrid/services/ActivitiesServiceIT.java | 2 +-
.../usergrid/services/CollectionServiceIT.java | 2 +
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 4 +-
.../usergrid/services/ServiceFactoryIT.java | 8 +-
.../usergrid/services/ServiceInvocationIT.java | 2 +-
.../usergrid/services/ServiceRequestIT.java | 9 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 4 +-
.../notifications/NotifiersServiceIT.java | 19 +-
.../apns/NotificationsServiceIT.java | 66 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../src/test/resources/log4j.properties | 6 +-
.../resources/usergrid-custom-test.properties | 2 +
.../usergrid/cassandra/SpringResource.java | 13 +-
.../setup/ConcurrentProcessSingleton.java | 49 +-
.../org/apache/usergrid/tools/AppAudit.java | 3 +-
.../org/apache/usergrid/tools/CredExport.java | 2 +-
.../org/apache/usergrid/tools/DupOrgRepair.java | 1 -
.../java/org/apache/usergrid/tools/Import.java | 14 +-
.../usergrid/tools/UniqueIndexCleanup.java | 6 +-
776 files changed, 30517 insertions(+), 39729 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 038be44,23514c8..748a069
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -70,11 -63,9 +63,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
- import org.apache.usergrid.persistence.cassandra.GeoIndexManager;
import org.apache.usergrid.persistence.cassandra.util.TraceParticipant;
- import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
- import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
++import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+ import org.apache.usergrid.persistence.collection.FieldSet;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@@ -88,15 -78,8 +79,9 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
import org.apache.usergrid.persistence.exceptions.RequiredPropertyNotFoundException;
import org.apache.usergrid.persistence.exceptions.UnexpectedEntityTypeException;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
- import org.apache.usergrid.persistence.index.EntityIndex;
- import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexScope;
- import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
- import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.index.query.Query.Level;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.model.entity.Id;
@@@ -178,21 -163,20 +165,24 @@@ public class CpEntityManager implement
public static final String APPLICATION_ENTITIES = "application.entities";
public static final long ONE_COUNT = 1L;
- private UUID applicationId;
+ private final UUID applicationId;
private Application application;
- private ManagerCache managerCache;
- private ApplicationScope applicationScope;
+ private final ManagerCache managerCache;
+
+ private final ApplicationScope applicationScope;
- private CassandraService cass;
+ private final CassandraService cass;
- private CounterUtils counterUtils;
+ private final CounterUtils counterUtils;
+
+ private final AsyncIndexService indexService;
+
+ private EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private GraphManagerFactory graphManagerFactory;
+
private boolean skipAggregateCounters;
private MetricsFactory metricsFactory;
private Timer aggCounterTimer;
@@@ -220,14 -207,22 +213,28 @@@
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
- public CpEntityManager() {
-
- }
-
- @Override
- public void init( final CassandraService cassandraService, final CounterUtils counterUtils, final MetricsFactory metricsFactory, final GraphManagerFactory graphManagerFactory, final EntityCollectionManagerFactory entityCollectionManagerFactory, final ManagerCache managerCache, UUID applicationId ) {
+ /**
+ * Fugly, make this part of DI
+ * @param cass
+ * @param counterUtils
+ * @param managerCache
+ * @param metricsFactory
+ * @param applicationId
++ * @param entityCollectionManagerFactory
++ * @param graphManagerFactory
+ */
- public CpEntityManager(final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache, final MetricsFactory metricsFactory, final UUID applicationId ) {
++ public CpEntityManager( final CassandraService cass, final CounterUtils counterUtils, final AsyncIndexService indexService, final ManagerCache managerCache,
++ final MetricsFactory metricsFactory,final EntityCollectionManagerFactory entityCollectionManagerFactory,
++ final GraphManagerFactory graphManagerFactory , final UUID applicationId ) {
++ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
++ this.graphManagerFactory = graphManagerFactory;
+ Preconditions.checkNotNull( cass, "cass must not be null" );
+ Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
+ Preconditions.checkNotNull( managerCache, "managerCache must not be null" );
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
+ Preconditions.checkNotNull( indexService, "indexService must not be null" );
+
this.managerCache = managerCache;
this.applicationId = applicationId;
@@@ -772,9 -733,10 +745,10 @@@
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
- CpRelationManager rmi = new CpRelationManager();
- rmi.init( this, applicationId, entityRef, null, metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory );
- return rmi;
+
+ CpRelationManager relationManager =
- new CpRelationManager( metricsFactory, managerCache, indexService, this, applicationId, entityRef );
++ new CpRelationManager( metricsFactory, managerCache, entityCollectionManagerFactory, graphManagerFactory, indexService, this, applicationId, entityRef );
+ return relationManager;
}
@@@ -2888,34 -2818,8 +2830,7 @@@
}
-
@Override
- public void refreshIndex() {
-
- // refresh factory indexes
-
-
- // refresh this Entity Manager's application's index
- EntityIndex ei = managerCache.getEntityIndex( getApplicationScope() );
- ei.refresh();
- }
-
-
- @Override
- public void createIndex() {
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- ei.initializeIndex();
- }
-
- public void deleteIndex(){
- EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- ei.deleteIndex();
- }
-
-
-
-
-
- @Override
public void flushManagerCaches() {
managerCache.invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 5b8e715,f08bce4..6ba71e2
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -177,10 -192,7 +192,8 @@@ public class CpEntityManagerFactory imp
private EntityManager _getEntityManager( UUID applicationId ) {
-
- EntityManager em = new CpEntityManager();
- em.init( cassandraService, counterUtils, metricsFactory, graphManagerFactory, entityCollectionManagerFactory, managerCache, applicationId );
-
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId );
++ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache, metricsFactory, applicationId,
++ entityCollectionManagerFactory, graphManagerFactory );
return em;
}
@@@ -432,17 -436,17 +437,17 @@@
logger.debug("getApplications(): Loading edges of edgeType {} from {}:{}",
new Object[] { edgeType, fromEntityId.getType(), fromEntityId.getUuid() } );
- Observable<Edge> edges = gm.loadEdgesFromSource(new SimpleSearchByEdgeType(
- fromEntityId, edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null));
+ Observable<Edge> edges = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
+ fromEntityId, edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ));
- Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
- while ( iter.hasNext() ) {
+ // TODO This is wrong, and will result in OOM if there are too many applications.
+ // This needs to stream properly with a buffer
- Edge edge = iter.next();
+ edges.doOnNext(edge -> {
Id targetId = edge.getTargetNode();
- logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[] {
+ logger.debug("getApplications(): Processing edge from {}:{} to {}:{}", new Object[]{
edge.getSourceNode().getType(), edge.getSourceNode().getUuid(),
edge.getTargetNode().getType(), edge.getTargetNode().getUuid()
});
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index edd352e,ea554a3..8e2fdd7
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -54,86 -50,44 +50,48 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.Schema;
import org.apache.usergrid.persistence.SimpleEntityRef;
import org.apache.usergrid.persistence.SimpleRoleRef;
- import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
- import org.apache.usergrid.persistence.cassandra.IndexUpdate;
- import org.apache.usergrid.persistence.cassandra.QueryProcessorImpl;
- import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner;
- import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
- import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
- import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
- import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
- import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
- import org.apache.usergrid.persistence.geo.ConnectionGeoSearch;
- import org.apache.usergrid.persistence.geo.EntityLocationRef;
- import org.apache.usergrid.persistence.geo.model.Point;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphManager;
+import org.apache.usergrid.persistence.graph.GraphManagerFactory;
+ import org.apache.usergrid.persistence.graph.SearchByEdge;
import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
- import org.apache.usergrid.persistence.index.EntityIndex;
+ import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
- import org.apache.usergrid.persistence.index.IndexScope;
+ import org.apache.usergrid.persistence.index.IndexEdge;
+ import org.apache.usergrid.persistence.index.SearchEdge;
import org.apache.usergrid.persistence.index.SearchTypes;
- import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.Identifier;
- import org.apache.usergrid.persistence.index.query.Query;
- import org.apache.usergrid.persistence.index.query.Query.Level;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
- import org.apache.usergrid.persistence.model.util.UUIDGenerator;
- import org.apache.usergrid.persistence.query.ir.AllNode;
- import org.apache.usergrid.persistence.query.ir.NameIdentifierNode;
- import org.apache.usergrid.persistence.query.ir.QueryNode;
- import org.apache.usergrid.persistence.query.ir.QuerySlice;
- import org.apache.usergrid.persistence.query.ir.SearchVisitor;
- import org.apache.usergrid.persistence.query.ir.WithinNode;
- import org.apache.usergrid.persistence.query.ir.result.ConnectionIndexSliceParser;
- import org.apache.usergrid.persistence.query.ir.result.ConnectionResultsLoaderFactory;
- import org.apache.usergrid.persistence.query.ir.result.ConnectionTypesIterator;
- import org.apache.usergrid.persistence.query.ir.result.EmptyIterator;
- import org.apache.usergrid.persistence.query.ir.result.GeoIterator;
- import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
- import org.apache.usergrid.persistence.query.ir.result.StaticIdIterator;
import org.apache.usergrid.persistence.schema.CollectionInfo;
- import org.apache.usergrid.utils.IndexUtils;
import org.apache.usergrid.utils.MapUtils;
- import org.apache.usergrid.utils.UUIDUtils;
import com.codahale.metrics.Timer;
+import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
- import me.prettyprint.hector.api.Keyspace;
- import me.prettyprint.hector.api.beans.DynamicComposite;
- import me.prettyprint.hector.api.beans.HColumn;
- import me.prettyprint.hector.api.mutation.Mutator;
import rx.Observable;
- import rx.functions.Action1;
- import rx.functions.Action2;
import rx.functions.Func1;
- import static java.util.Arrays.asList;
-
- import static me.prettyprint.hector.api.factory.HFactory.createMutator;
- import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionSearchEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchByEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchEdge;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createSearchEdgeFromSource;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.generateScopeFromSource;
+ import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getNameFromEdgeType;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_ENTITIES;
- import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_TYPES;
- import static org.apache.usergrid.persistence.Schema.INDEX_CONNECTIONS;
- import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.PROPERTY_TITLE;
@@@ -172,10 -106,7 +110,9 @@@ public class CpRelationManager implemen
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
-
private ManagerCache managerCache;
+ private EntityCollectionManagerFactory entityCollectionManagerFactory;
+ private GraphManagerFactory graphManagerFactory;
private EntityManager em;
@@@ -194,22 -122,10 +128,11 @@@
private MetricsFactory metricsFactory;
private Timer updateCollectionTimer;
- private Timer createConnectionTimer;
- private Timer cassConnectionDelete;
- private Timer esDeleteConnectionTimer;
- public CpRelationManager() {}
- public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
++ public CpRelationManager( final MetricsFactory metricsFactory, final ManagerCache managerCache, final
++ EntityCollectionManagerFactory entityCollectionManagerFactory, final GraphManagerFactory graphManagerFactory, final AsyncIndexService indexService, final EntityManager em, final UUID applicationId, final EntityRef headEntity) {
- public CpRelationManager init(
- EntityManager em,
- UUID applicationId,
- EntityRef headEntity,
- IndexBucketLocator indexBucketLocator,
- MetricsFactory metricsFactory,
- ManagerCache managerCache,
- EntityCollectionManagerFactory entityCollectionManagerFactory,
- GraphManagerFactory graphManagerFactory ) {
Assert.notNull( em, "Entity manager cannot be null" );
Assert.notNull( applicationId, "Application Id cannot be null" );
@@@ -225,30 -141,13 +148,16 @@@
this.managerCache = managerCache;
this.applicationScope = CpNamingUtils.getApplicationScope( applicationId );
- this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence
- this.indexBucketLocator = indexBucketLocator; // TODO: this also
- this.metricsFactory = metricsFactory;
- this.updateCollectionTimer = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.es.update.collection" );
- this.createConnectionTimer = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.es.create.connection.timer" );
- this.cassConnectionDelete = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.cassandra.delete.connection.batch.timer" );
- this.esDeleteConnectionTimer = metricsFactory.getTimer(CpRelationManager.class, "relation.manager.es.delete.connection.batch.timer" );
- // load the Core Persistence version of the head entity as well
- this.headEntityScope = getCollectionScopeNameFromEntityType(
- applicationScope.getApplication(), headEntity.getType());
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ this.graphManagerFactory = graphManagerFactory;
+
+ this.metricsFactory = metricsFactory;
+ this.updateCollectionTimer =
+ metricsFactory.getTimer( CpRelationManager.class, "relation.manager.es.update.collection" );
+
if ( logger.isDebugEnabled() ) {
- logger.debug( "Loading head entity {}:{} from scope\n app {}\n owner {}\n name {}",
- new Object[] {
- headEntity.getType(),
- headEntity.getUuid(),
- headEntityScope.getApplication(),
- headEntityScope.getOwner(),
- headEntityScope.getName()
+ logger.debug( "Loading head entity {}:{} from app {}", new Object[] {
+ headEntity.getType(), headEntity.getUuid(), applicationScope
} );
}
@@@ -339,8 -227,7 +237,7 @@@
public Observable<Edge> call( final String edgeType ) {
return gm.loadEdgesToTarget(
new SimpleSearchByEdgeType( cpHeadEntity.getId(), edgeType, Long.MAX_VALUE,
- SearchByEdgeType.Order.DESCENDING, null ) );
+ SearchByEdgeType.Order.DESCENDING, Optional.<Edge>absent() ) );
-
}
} );
@@@ -538,24 -295,27 +305,6 @@@
}
- private boolean moreThanOneOutboundConnection( EntityRef source, String connectionType ) {
- private boolean moreThanOneInboundConnection( EntityRef target, String connectionType ) {
--
- Id sourceId = new SimpleId( source.getUuid(), source.getType() );
- Id targetId = new SimpleId( target.getUuid(), target.getType() );
--
-- GraphManager gm = managerCache.getGraphManager( applicationScope );
-
- Observable<Edge> edgesFromSource = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- sourceId,
- CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
- System.currentTimeMillis(),
- SearchByEdgeType.Order.DESCENDING,
- Optional.<Edge>absent() ) ); // last
--
- int count = edgesFromSource.take( 2 ).count().toBlocking().last();
- Observable<Edge> edgesToTarget = gm.loadEdgesToTarget(
- new SimpleSearchByEdgeType( targetId, CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
- System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) ); // last
--
- return count > 1;
- Iterator<Edge> iterator = edgesToTarget.toBlocking().getIterator();
- int count = 0;
- while ( iterator.hasNext() ) {
- iterator.next();
- if ( count++ > 1 ) {
- return true;
- }
- }
- return false;
-- }
--
@Override
public Set<String> getCollections() throws Exception {
@@@ -927,43 -637,29 +626,31 @@@
headEntity = em.validate( headEntity );
- CollectionInfo collection =
- getDefaultSchema().getCollection( headEntity.getType(), collName );
+ CollectionInfo collection = getDefaultSchema().getCollection( headEntity.getType(), collName );
if ( collection == null ) {
- throw new RuntimeException( "Cannot find collection-info for '" + collName
- + "' of " + headEntity.getType() + ":" + headEntity .getUuid() );
+ throw new RuntimeException(
+ "Cannot find collection-info for '" + collName + "' of " + headEntity.getType() + ":" + headEntity
+ .getUuid() );
}
- query.setEntityType( collection.getType() );
- query = adjustQuery( query );
- /**
- * It's a graph query, execute the graph query executor.
- *
- * TODO refactor all of this away into commands
- */
- if(query.isGraphQuery()){
- final QueryExecutor executor = new CollectionGraphQueryExecutor( this.entityCollectionManagerFactory,graphManagerFactory, applicationScope, headEntity, query.getCursor(), collName, query.getLimit() );
- return executor.next();
- }
+ final SearchEdge searchEdge = createCollectionSearchEdge( cpHeadEntity.getId(), collName );
+ final ApplicationEntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
- final IndexScope indexScope = new IndexScopeImpl(
- cpHeadEntity.getId(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
-
- final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+ logger.debug( "Searching scope {}", searchEdge );
- final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
+ query.setEntityType( collection.getType() );
+ query = adjustQuery( query );
- logger.debug( "Searching scope {}:{}",
- indexScope.getOwner().toString(), indexScope.getName() );
+
+
- final CollectionResultsLoaderFactoryImpl resultsLoaderFactory = new CollectionResultsLoaderFactoryImpl( managerCache );
+ final CollectionResultsLoaderFactoryImpl resultsLoaderFactory =
+ new CollectionResultsLoaderFactoryImpl( managerCache );
//execute the query and return our next result
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index b2354a6,3f2c9d6..15e4341
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@@ -103,56 -103,39 +105,39 @@@ public class CpWalker
edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
}
- Observable<String> edgeTypes = gm.getEdgeTypesFromSource(
- new SimpleSearchEdgeType( applicationId, edgeType, null ) );
-
- edgeTypes.flatMap( new Func1<String, Observable<Edge>>() {
- @Override
- public Observable<Edge> call( final String edgeType ) {
-
- logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
-
- return gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- applicationId, edgeType, Long.MAX_VALUE, order , Optional.<Edge>absent() ) );
-
- }
-
- } ).parallel( new Func1<Observable<Edge>, Observable<Edge>>() {
-
- @Override
- public Observable<Edge> call( final Observable<Edge> edgeObservable ) { // process edges in parallel
- return edgeObservable.doOnNext( new Action1<Edge>() { // visit and update then entity
-
- @Override
- public void call( Edge edge ) {
-
- logger.info( "Re-indexing edge {}", edge );
-
- EntityRef targetNodeEntityRef = new SimpleEntityRef(
- edge.getTargetNode().getType(),
- edge.getTargetNode().getUuid() );
-
- Entity entity;
- try {
- entity = em.get( targetNodeEntityRef );
- }
- catch ( Exception ex ) {
- logger.error( "Error getting sourceEntity {}:{}, continuing",
- targetNodeEntityRef.getType(),
- targetNodeEntityRef.getUuid() );
- return;
- }
- if(entity == null){
- return;
- }
- String collName = CpNamingUtils.getCollectionName( edge.getType() );
- visitor.visitCollectionEntry( em, collName, entity );
- }
- } );
- }
- }, Schedulers.io() )
+ Observable<Edge> edges = gm.getEdgeTypesFromSource(
+ new SimpleSearchEdgeType( applicationId, edgeType, null ) ).flatMap( emittedEdgeType -> {
+
+ logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
+
+ return gm.loadEdgesFromSource(
- new SimpleSearchByEdgeType( applicationId, emittedEdgeType, Long.MAX_VALUE, order, null ) );
++ new SimpleSearchByEdgeType( applicationId, emittedEdgeType, Long.MAX_VALUE, order, Optional.absent() ) );
+ } ).flatMap( edge -> {
+ //run each edge through it's own scheduler, up to 100 at a time
+ return Observable.just( edge ).doOnNext( edgeValue -> {
+ logger.info( "Re-indexing edge {}", edgeValue );
+
+ EntityRef targetNodeEntityRef =
+ new SimpleEntityRef( edgeValue.getTargetNode().getType(), edgeValue.getTargetNode().getUuid() );
+
+ Entity entity;
+ try {
+ entity = em.get( targetNodeEntityRef );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error getting sourceEntity {}:{}, continuing", targetNodeEntityRef.getType(),
+ targetNodeEntityRef.getUuid() );
+ return;
+ }
+ if ( entity == null ) {
+ return;
+ }
+ String collName = getNameFromEdgeType( edgeValue.getType() );
+ visitor.visitCollectionEntry( em, collName, entity );
+ } ).subscribeOn( Schedulers.io() );
+ }, 100 );
// wait for it to complete
- .toBlocking().lastOrDefault( null ); // end foreach on edges
+ edges.toBlocking().lastOrDefault( null ); // end foreach on edges
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ec3a710a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
[19/20] incubator-usergrid git commit: Removed all field mapping
Posted by to...@apache.org.
Removed all field mapping
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/223c4b39
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/223c4b39
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/223c4b39
Branch: refs/heads/two-dot-o-dev
Commit: 223c4b398a3894de319cdfa5a2fe95acb059bcd0
Parents: 23acdd5
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 27 11:38:17 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 27 11:38:17 2015 -0600
----------------------------------------------------------------------
.../persistence/index/usergrid-mappings.json | 171 ++++++++++---------
.../setup/ConcurrentProcessSingleton.java | 6 +-
2 files changed, 89 insertions(+), 88 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/223c4b39/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
index d4c34cd..f12b66b 100644
--- a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
+++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
@@ -1,93 +1,96 @@
{
- "entity": {
- "properties": {
- "entityId": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "entityVersion": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "entityType": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "applicationId": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "nodeId": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "edgeName": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "entityNodeType": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "edgeTimestamp": {
- "type": "long",
- "doc_values": true
- },
- "edgeSearch": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "fields": {
- "type": "nested",
+ "entity": {
"properties": {
- "name": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- },
- "boolean": {
- "type": "boolean",
- "doc_values": true
- },
- "long": {
- "type": "long",
- "doc_values": true
- },
- "double": {
- "type": "double",
- "doc_values": true
- },
- "location": {
- "type": "geo_point",
- "lat_lon": true,
- "geohash": true,
- "doc_values": true
- },
- "string": {
- "type": "string",
- "fields": {
- "exact": {
+ "entityId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "entityVersion": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "entityType": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "applicationId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "nodeId": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "edgeName": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "entityNodeType": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "edgeTimestamp": {
+ "type": "long",
+ "doc_values": true
+ },
+ "edgeSearch": {
"type": "string",
"index": "not_analyzed",
"doc_values": true
- }
+ },
+ "fields": {
+ "type": "nested",
+ "properties": {
+ "name": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ },
+ "boolean": {
+ "type": "boolean",
+ "doc_values": true
+ },
+ "long": {
+ "type": "long",
+ "doc_values": true
+ },
+ "double": {
+ "type": "double",
+ "doc_values": true
+ },
+ "location": {
+ "type": "geo_point",
+ "lat_lon": true,
+ "geohash": true,
+ "doc_values": true
+ },
+ "string": {
+ "type": "string",
+ "fields": {
+ "exact": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ }
+ }
+ },
+ "uuid": {
+ "type": "string",
+ "index": "not_analyzed",
+ "doc_values": true
+ }
+ }
}
- },
- "uuid": {
- "type": "string",
- "index": "not_analyzed",
- "doc_values": true
- }
}
- }
+ },
+ "_all": {
+ "enabled": false
}
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/223c4b39/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java b/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
index 214334b..7cb5073 100644
--- a/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/setup/ConcurrentProcessSingleton.java
@@ -43,10 +43,8 @@ public class ConcurrentProcessSingleton {
public static final int LOCK_PORT = Integer.parseInt(
System.getProperty( "test.lock.port", "10101" ) );
-// public static final boolean CLEAN_STORAGE =
-// Boolean.parseBoolean( System.getProperty( "test.clean.storage", "false" ) );
-
- public static final boolean CLEAN_STORAGE = false;
+ public static final boolean CLEAN_STORAGE =
+ Boolean.parseBoolean( System.getProperty( "test.clean.storage", "false" ) );
public static final long ONE_MINUTE = 60000;