You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/27 19:39:24 UTC

[15/20] incubator-usergrid git commit: First pass at new command layout. Need to test and debug

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
deleted file mode 100644
index 58e4296..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/elasticsearch/impl/VersionVerifier.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.elasticsearch.impl;
-
-
-import java.util.Collection;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.VersionSet;
-import org.apache.usergrid.persistence.index.CandidateResult;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.fasterxml.uuid.UUIDComparator;
-
-
-/**
- * A loader that verifies versions are correct in Cassandra and match ElasticSearch
- */
-public abstract class VersionVerifier implements ResultsVerifier {
-
-    private static final Logger logger = LoggerFactory.getLogger( VersionVerifier.class );
-
-    private VersionSet ids;
-
-
-    @Override
-    public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
-        ids = ecm.getLatestVersion( idsToLoad ).toBlocking().last();
-    }
-
-
-    @Override
-    public boolean isValid( final CandidateResult candidateResult ) {
-        final Id entityId = candidateResult.getId();
-
-        final MvccLogEntry version = ids.getMaxVersion( entityId );
-
-        //version wasn't found ,deindex
-        if ( version == null ) {
-            logger.warn( "Version for Entity {}:{} not found",
-                    entityId.getUuid(), entityId.getUuid() );
-
-            return false;
-        }
-
-        final UUID savedVersion = version.getVersion();
-
-        if ( UUIDComparator.staticCompare( savedVersion, candidateResult.getVersion() ) > 0 ) {
-            logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
-                new Object[] {
-                    entityId.getUuid(),
-                    entityId.getType(),
-                    candidateResult.getVersion(),
-                    savedVersion
-            } );
-
-            return false;
-        }
-
-        return true;
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
deleted file mode 100644
index 1504f26..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/entity/EntityLoadCollector.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.entity;
-
-
-import java.io.Serializable;
-import java.util.Map;
-
-import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
-import org.apache.usergrid.corepersistence.command.read.Collector;
-import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-
-
-/**
- * Loads entities from a set of Ids.
- *
- * TODO refactor this into a common command that both ES search and graphSearch can use for repair and verification
- */
-public class EntityLoadCollector extends AbstractCommand<Results, Serializable> implements Collector<Results> {
-
-    private final EntityCollectionManagerFactory entityCollectionManagerFactory;
-
-    //TODO get rid of this when merged into 2.0 dev
-    private final ApplicationScope applicationScope;
-    private int resultSize;
-
-
-    public EntityLoadCollector( final EntityCollectionManagerFactory entityCollectionManagerFactory,
-                                final ApplicationScope applicationScope ) {
-        this.entityCollectionManagerFactory = entityCollectionManagerFactory;
-        this.applicationScope = applicationScope;
-    }
-
-
-    @Override
-    protected Class<Serializable> getCursorClass() {
-        return null;
-    }
-
-
-    @Override
-    public Observable<Results> call( final Observable<Id> observable ) {
-
-
-        /**
-         * A bit kludgy from old 1.0 -> 2.0 apis.  Refactor this as we clean up our lower levels and create new results
-         * objects
-         */
-
-        final EntityCollectionManager entityCollectionManager =
-            entityCollectionManagerFactory.createCollectionManager( applicationScope );
-
-        final Observable<EntitySet> entitySetObservable = observable.buffer( resultSize ).flatMap(
-            bufferedIds -> Observable.just( bufferedIds ).flatMap( ids -> entityCollectionManager.load( ids ) ) );
-
-
-        final Observable<Results> resultsObservable =  entitySetObservable
-
-            .flatMap( entitySet -> {
-
-            //get our entites and filter missing ones, then collect them into a results object
-            final Observable<MvccEntity> mvccEntityObservable = Observable.from( entitySet.getEntities() );
-
-            //convert them to our old entity model, then filter nulls, meaning they weren't found
-            return mvccEntityObservable.map( mvccEntity -> mapEntity( mvccEntity ) ).filter( entity -> entity == null )
-
-                //convert them to a list, then map them into results
-                .toList().map( entities -> {
-                    final Results results = Results.fromEntities( entities );
-                    results.setCursor( generateCursor() );
-
-                    return results;
-                } )
-                //if no results are present, return an empty results
-                .singleOrDefault( new Results(  ) );
-        } );
-
-
-        return resultsObservable;
-    }
-
-                /**
-                 * Map a new cp entity to an old entity.  May be null if not present
-                 */
-
-
-    private org.apache.usergrid.persistence.Entity mapEntity( final MvccEntity mvccEntity ) {
-        if ( !mvccEntity.getEntity().isPresent() ) {
-            return null;
-        }
-
-
-        final Entity cpEntity = mvccEntity.getEntity().get();
-        final Id entityId = cpEntity.getId();
-
-        org.apache.usergrid.persistence.Entity entity =
-            EntityFactory.newEntity( entityId.getUuid(), entityId.getType() );
-
-        Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
-        entity.addProperties( entityMap );
-
-        return entity;
-    }
-
-
-    @Override
-    public void setLimit( final int limit ) {
-        this.resultSize = limit;
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
deleted file mode 100644
index 95b6c6b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/AbstractReadGraphCommand.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.graph;
-
-
-import org.apache.usergrid.corepersistence.command.read.AbstractCommand;
-import org.apache.usergrid.corepersistence.command.read.TraverseCommand;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Optional;
-
-import rx.Observable;
-import rx.functions.Action1;
-import rx.functions.Func1;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
-
-
-/**
- * Command for reading graph edges
- */
-public abstract class AbstractReadGraphCommand extends AbstractCommand<Id, Edge> implements TraverseCommand {
-
-    private final GraphManagerFactory graphManagerFactory;
-
-
-    /**
-     * Create a new instance of our command
-     * @param graphManagerFactory
-     */
-    public AbstractReadGraphCommand( final GraphManagerFactory graphManagerFactory ) {
-        this.graphManagerFactory = graphManagerFactory;
-    }
-
-
-    @Override
-    public Observable<Id> call( final Observable<Id> observable ) {
-
-        //get the graph manager
-        final GraphManager graphManager = graphManagerFactory.createEdgeManager( applicationScope );
-
-        //set our our constant state
-        final Optional<Edge> startFromCursor = getCursor();
-
-        final String edgeName = getEdgeTypeName();
-
-
-        //return all ids that are emitted from this edge
-        return observable.flatMap( new Func1<Id, Observable<Id>>() {
-
-            @Override
-            public Observable<Id> call( final Id id ) {
-
-                final SimpleSearchByEdgeType search = new SimpleSearchByEdgeType(id,edgeName, Long.MAX_VALUE,
-                    SearchByEdgeType.Order.DESCENDING, startFromCursor   );
-
-                /**
-                 * TODO, pass a message with pointers to our cursor values to be generated later
-                 */
-                return graphManager.loadEdgesFromSource( search ).doOnNext( new Action1<Edge>() {
-                    @Override
-                    public void call( final Edge edge ) {
-                        setCursor( edge );
-                    }
-                } ).map( new Func1<Edge, Id>() {
-                    @Override
-                    public Id call( final Edge edge ) {
-                        return edge.getTargetNode();
-                    }
-                } );
-            }
-        } );
-    }
-
-
-    @Override
-    protected Class<Edge> getCursorClass() {
-        return Edge.class;
-    }
-
-
-
-    /**
-     * Get the edge type name we should use when traversing
-     * @return
-     */
-    protected abstract String getEdgeTypeName();
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
deleted file mode 100644
index 3b6633b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphCollectionCommand.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.graph;
-
-
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.inject.Inject;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromCollectionName;
-
-
-/**
- * Command for reading graph edges on a collection
- */
-public class ReadGraphCollectionCommand extends AbstractReadGraphCommand {
-
-    private final String collectionName;
-
-
-    /**
-     * Create a new instance of our command
-     */
-    @Inject
-    public ReadGraphCollectionCommand( final GraphManagerFactory graphManagerFactory, final String collectionName ) {
-        super( graphManagerFactory );
-        this.collectionName = collectionName;
-    }
-
-
-    @Override
-    protected String getEdgeTypeName() {
-        return getCollectionScopeNameFromCollectionName( collectionName );
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
deleted file mode 100644
index 26b30e7..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/command/read/graph/ReadGraphConnectionCommand.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.command.read.graph;
-
-
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-
-import com.google.inject.Inject;
-
-import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getConnectionScopeName;
-
-
-/**
- * Command for reading graph edges on a connection
- */
-public class ReadGraphConnectionCommand extends AbstractReadGraphCommand {
-
-    private final String connectionName;
-
-
-    /**
-     * Create a new instance of our command
-     */
-    @Inject
-    public ReadGraphConnectionCommand( final GraphManagerFactory graphManagerFactory, final String connectionName ) {
-        super( graphManagerFactory );
-        this.connectionName = connectionName;
-    }
-
-
-    @Override
-    protected String getEdgeTypeName() {
-        return getConnectionScopeName( connectionName );
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java
new file mode 100644
index 0000000..8463df9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/DataPipeline.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.Filter;
+import org.apache.usergrid.corepersistence.pipeline.read.TraverseFilter;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * A pipeline that will allow us to build a traversal command for execution
+ *
+ * See http://martinfowler.com/articles/collection-pipeline/ for some examples
+ *
+ * TODO: Re work the cursor and limit phases.  They need to be lazily evaluated, not added on build time
+ */
+public class DataPipeline {
+
+
+    private final ApplicationScope applicationScope;
+    private final List<TraverseFilter> traverseFilterList;
+
+    private Optional<String> cursor;
+    private int limit;
+
+
+    private int count = 0;
+
+
+    /**
+     * Our first pass, where we implement our start point as an Id until we can use this to perform our entire
+     * traversal.  Eventually as we untangle the existing Query service nightmare, the sourceId will be remove and
+     * should only be traversed from the root application
+     */
+    public DataPipeline( final ApplicationScope applicationScope ) {
+
+        this.applicationScope = applicationScope;
+
+
+        traverseFilterList = new ArrayList<>();
+    }
+
+
+    /**
+     * Add a read command that will read Ids and produce Ids.  This is an intermediate traversal operations
+     */
+    public DataPipeline withTraverseCommand( final TraverseFilter traverseCommand ) {
+
+        this.traverseFilterList.add( traverseCommand );
+
+        return this;
+    }
+
+
+    /**
+     * Build the final collection step, and process our filters
+     */
+    public <T> Observable<T> build( final CollectorFilter<T> pipeCollector ) {
+
+        RequestCursor requestCursor = new RequestCursor( this.cursor );
+        ResponseCursor responseCursor = new ResponseCursor();
+
+        Observable<Id> traverseObservable = Observable.just( applicationScope.getApplication() );
+
+        //build our traversal commands
+        for ( TraverseFilter filter : traverseFilterList ) {
+            setState( filter, requestCursor, responseCursor );
+
+            traverseObservable = traverseObservable.compose( filter );
+        }
+
+
+        setState( pipeCollector, requestCursor, responseCursor );
+
+        pipeCollector.setLimit( limit );
+
+        return traverseObservable.compose( pipeCollector );
+    }
+
+
+    public void setCursor( Optional<String> cursor ) {
+        this.cursor = cursor;
+    }
+
+
+    public void setLimit( final int limit ) {
+        this.limit = limit;
+    }
+
+
+    /**
+     * Set the id of the state
+     */
+    private void setState( final Filter<?> filter, final RequestCursor requestCursor,
+                           final ResponseCursor responseCursor ) {
+
+        //TODO, see if we can wrap this observable in our ObservableTimer so we can see how long each filter takes
+
+
+        filter.setId( count );
+        //done for clarity
+        count++;
+
+        filter.setCursorCaches( requestCursor, responseCursor );
+        filter.setApplicationScope( applicationScope );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
new file mode 100644
index 0000000..9916bc1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineBuilderFactory.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+
+/**
+ * Factory for creating pipeline builders
+ */
+public interface PipelineBuilderFactory {
+
+
+    /**
+     * Create a read pipeline builder
+     * @param applicationScope
+     * @return
+     */
+    ReadPipelineBuilder createReadPipelineBuilder( final ApplicationScope applicationScope );
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
new file mode 100644
index 0000000..8afe2f8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/PipelineModule.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.ReadPipelineBuilder;
+
+import com.google.inject.AbstractModule;
+import com.google.inject.assistedinject.FactoryModuleBuilder;
+
+
+/**
+ * Configures all our pipelines
+ */
+public class PipelineModule extends AbstractModule {
+
+    @Override
+    protected void configure() {
+        //Use Guice to create the builder since we don't really need to do anything
+        //other than DI when creating the filters
+       install(new FactoryModuleBuilder().build( ReadPipelineBuilder.class ));
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java
new file mode 100644
index 0000000..ce2e10a
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/CursorSerializer.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.cursor;
+
+
+import java.io.Serializable;
+
+import com.fasterxml.jackson.core.Base64Variant;
+import com.fasterxml.jackson.core.Base64Variants;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.smile.SmileFactory;
+
+
+/**
+ * A utility to serialize objects to/from cursors
+ */
+public class CursorSerializer {
+
+
+    private static final SmileFactory SMILE_FACTORY = new SmileFactory();
+
+    private static final ObjectMapper MAPPER = new ObjectMapper( SMILE_FACTORY );
+
+    private static final Base64Variant VARIANT = Base64Variants.MODIFIED_FOR_URL;
+
+
+    /**
+     * Serialize the serializable object as a cursor
+     */
+    public static String asCursor( final Serializable cursor ) {
+
+        try {
+            return MAPPER.writer( VARIANT ).writeValueAsString( cursor );
+        }
+        catch ( JsonProcessingException e ) {
+            throw new CursorParseException( "Unable to serialize cursor", e );
+        }
+    }
+
+
+    /**
+     * Deserialize from the cursor
+     * @param cursor
+     * @return
+     * @throws CursorParseException
+     */
+    public <T extends Serializable> T fromCursor( final String cursor, final Class<T> cursorClass ) throws CursorParseException {
+        try {
+
+            final JsonParser parser = MAPPER.getFactory().createParser( cursor );
+            return MAPPER.reader( VARIANT ).readValue( parser, cursorClass);
+        }
+        catch ( Exception e ) {
+            throw new CursorParseException( "Unable to serialize cursor", e );
+        }
+    }
+
+
+    /**
+     * Thrown when we can't parse a cursor
+     */
+    public static class CursorParseException extends RuntimeException {
+        public CursorParseException( final String message, final Throwable cause ) {
+            super( message, cause );
+        }
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
new file mode 100644
index 0000000..99cfd74
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/RequestCursor.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.cursor;
+
+
+import java.io.Serializable;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A cursor that has been passed in with our request.  Adds utils for parsing values
+ */
+public class RequestCursor {
+
+    public RequestCursor(final Optional<String> cursor){
+
+    }
+
+
+    /**
+     * Get the cursor with the specified id
+     */
+    public <T extends Serializable> T getCursor( final int id, final Class<T> cursorType ) {
+        return null;
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
new file mode 100644
index 0000000..0e047ed
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/cursor/ResponseCursor.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.cursor;
+
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Map;
+
+
+/**
+ * A cursor used in rendering a response
+ */
+public class ResponseCursor {
+
+
+
+    /**
+     * We use a map b/c some indexes might be skipped
+     */
+    private Map<Integer, ? super Serializable> cursors = new HashMap<>();
+
+    /**
+     * Set the possible cursor value into the index. DOES NOT parse the cursor.  This is intentional for performance
+     */
+    public <T extends Serializable> void setCursor( final int id, final T cursor ) {
+        cursors.put( id, cursor );
+    }
+
+
+    private void ensureCapacity() {
+
+    }
+
+
+    public String encodeAsString() {
+        return null;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
new file mode 100644
index 0000000..a6e7ef1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/AbstractFilter.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import java.io.Serializable;
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * Basic functionality for our commands to handle cursor IO
+ */
+public abstract class AbstractFilter<T, C extends Serializable> implements Filter<T> {
+
+    private int id;
+    /**
+     * The cache of the cursor that was set when the read was started
+     */
+    private RequestCursor readCache;
+
+    /**
+     * The current state of the write cache.  Gets updated as we traverse the observables
+     */
+    private ResponseCursor writeCache;
+
+
+    /**
+     * The applicationScope
+     */
+    protected ApplicationScope applicationScope;
+
+
+    @Override
+    public void setId( final int id ) {
+        this.id = id;
+    }
+
+
+    @Override
+    public void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache ) {
+        this.readCache = readCache;
+        this.writeCache = writeCache;
+    }
+
+
+    @Override
+    public void setApplicationScope( final ApplicationScope applicationScope ) {
+       this.applicationScope = applicationScope;
+    }
+
+
+    /**
+     * Return the parsed value of the cursor from the last request, if it exists
+     */
+    protected Optional<C> getCursor() {
+        final C cursor = readCache.getCursor( id, getCursorClass() );
+
+        return Optional.fromNullable( cursor );
+    }
+
+
+
+
+
+    /**
+     * Set the cursor value into the new cursor write cache
+     * @param newValue
+     */
+    protected void setCursor(final C newValue){
+        writeCache.setCursor( id, newValue );
+    }
+
+
+    /**
+     * Generate our state as a cursor
+     * @return
+     */
+    protected String generateCursor(){
+        return writeCache.encodeAsString();
+    }
+
+    /**
+     * Return the class to be used when parsing the cursor
+     */
+    protected abstract Class<C> getCursorClass();
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java
new file mode 100644
index 0000000..883e910
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/CollectorFilter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+/**
+ * A command that is used to reduce our stream of results into a final output
+ * @param <T>
+ */
+public interface CollectorFilter<T> extends Filter<T> {
+
+    /**
+     * Set the prefered result size for the command
+     * @param limit
+     */
+    void setLimit( final int limit );
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
new file mode 100644
index 0000000..670d2ed
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/Filter.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.corepersistence.pipeline.cursor.RequestCursor;
+import org.apache.usergrid.corepersistence.pipeline.cursor.ResponseCursor;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+/**
+ * Interface for filtering commands.  All filters must take an observable of Id's as an input.  Output is then determined by subclasses.
+ * This takes an input of Id, performs some operation, and emits values for further processing in the Observable
+ * pipeline
+ */
+public interface Filter<T> extends Observable.Transformer<Id, T> {
+
+
+    /**
+     * Set the id of this command in it's execution environment
+     */
+    void setId( final int id );
+
+    /**
+     * Set the cursor cache into the command
+     *
+     * @param readCache Set the cache that was used in the request
+     * @param writeCache Set the cache to be used when writing the results
+     */
+    void setCursorCaches( final RequestCursor readCache, final ResponseCursor writeCache );
+
+    /**
+     * Set the application scope of the command
+     * @param applicationScope
+     */
+    void setApplicationScope(final ApplicationScope applicationScope);
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
new file mode 100644
index 0000000..525678c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/FilterFactory.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryCollectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.QueryConnectionElasticSearchCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphCollectionFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByIdFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionByTypeFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.graph.ReadGraphConnectionFilter;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A factory for generating read commands
+ */
+public interface FilterFactory {
+
+
+    /**
+     * Generate a new instance of the command with the specified parameters
+     */
+    ReadGraphCollectionFilter readGraphCollectionCommand( final String collectionName );
+
+    /**
+     * Read a connection between two entities, the incoming and the target entity
+     * @param collectionName
+     * @param targetId
+     * @return
+     */
+    ReadGraphCollectionByIdFilter readGraphCollectionByIdFilter(final String collectionName, final Id targetId);
+
+    /**
+     * Generate a new instance of the command with the specified parameters
+     */
+    ReadGraphConnectionFilter readGraphConnectionCommand( final String connectionName );
+
+    /**
+     * Generate a new instance of the command with the specified parameters
+     */
+    ReadGraphConnectionByTypeFilter readGraphConnectionCommand( final String connectionName, final String entityType );
+
+
+    /**
+     * Read a connection directly between two identifiers
+     * @param connectionName
+     * @param targetId
+     * @return
+     */
+    ReadGraphConnectionByIdFilter readGraphConnectionByIdFilter(final String connectionName, final Id targetId);
+
+    /**
+     * Generate a new instance of the command with the specified parameters
+     */
+    EntityLoadCollectorFilter entityLoadCollector();
+
+    /**
+     * Generate a new instance of the command with the specified parameters
+     */
+    QueryCollectionElasticSearchCollectorFilter queryCollectionElasticSearchCollector( final String collectionName, final String query,
+                                                                                       final Optional<String> cursor, final int limit );
+
+
+    /**
+     * Generate a new instance of the command with the specified parameters
+     */
+    QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName,final String query,
+                                                                                       final Optional<String> cursor, final int limit );
+
+
+    /**
+     * Generate a new instance of the command with the specified parameters
+     */
+    QueryConnectionElasticSearchCollectorFilter queryConnectionElasticSearchCollector( final String connectionName, final String connectionEntityType, final String query,
+                                                                                       final Optional<String> cursor, final int limit );
+
+
+    /**
+     * Get an entity id filter.  Used as a 1.0->2.0 bridge since we're not doing full traversals
+     */
+    EntityIdFilter getEntityIdFilter( final Id entityId );
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
new file mode 100644
index 0000000..13aa3eb
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilder.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+
+import rx.Observable;
+
+
+/**
+ * An instance of a pipeline builder for building commands.
+ * Each invocation of the method will assemple the underlying pipe and updating it's state
+ * Results are added by invoking execute.
+ */
+public interface ReadPipelineBuilder {
+
+    /**
+     * An operation to bridge 2.0-> 1.0.  Should be removed when everyone uses the pipeline
+     * @param id
+     * @return
+     */
+    ReadPipelineBuilder setStartId(final Id id);
+
+    /**
+     * Add a get entity to the pipeline
+     */
+    ReadPipelineBuilder getEntityViaCollection( final String collectionName, final Id entityId );
+
+
+    /**
+     * Add get Collection from our previous source
+     */
+    ReadPipelineBuilder getCollection( final String collectionName );
+
+    /**
+     * Get all entities with a query
+     */
+    ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query, final int limit,
+                                 final Optional<String> cursor );
+
+    /**
+     * Get an entity via the connection name and entity Id
+     */
+    ReadPipelineBuilder getEntityViaConnection( final String connectionName, final Id entityId );
+
+    /**
+     * Get all entities in a connection by the connection name
+     */
+    ReadPipelineBuilder getConnection( final String connectionName );
+
+    /**
+     * Get all entities in a connection of the specified connection type
+     */
+    ReadPipelineBuilder getConnection( final String connectionName, final String entityType );
+
+    /**
+     * Get all entities in a connection with a query
+     */
+    ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query, final int limit,
+                              final Optional<String> cursor );
+
+
+    /**
+     * Get all entities in a connection with a query
+     */
+    ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType, final String query, final int limit,
+                              final Optional<String> cursor );
+
+
+
+
+
+
+    /**
+     * Execute final construction of the pipeline and return the results
+     * @return
+     */
+    Observable<Results> execute();
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
new file mode 100644
index 0000000..a44f4b8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/ReadPipelineBuilderImpl.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import java.util.Collections;
+import java.util.UUID;
+
+import org.apache.usergrid.corepersistence.pipeline.DataPipeline;
+import org.apache.usergrid.corepersistence.pipeline.read.entity.EntityLoadCollectorFilter;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import rx.Observable;
+
+
+/**
+ * An implementation of our builder for piplines
+ */
+public class ReadPipelineBuilderImpl implements ReadPipelineBuilder {
+
+
+    private final FilterFactory filterFactory;
+
+    private final DataPipeline pipeline;
+
+    /**
+     * Our pointer to our collect filter. Set or cleared with each operation that's performed so the correct
+     * results are rendered
+     */
+    private CollectorFilter<Results> collectorFilter;
+
+
+    @Inject
+    public ReadPipelineBuilderImpl( final FilterFactory filterFactory,
+                                    @Assisted final ApplicationScope applicationScope ) {
+        this.filterFactory = filterFactory;
+        this.pipeline = new DataPipeline( applicationScope );
+    }
+
+
+    @Override
+    public ReadPipelineBuilder setStartId( final Id id ) {
+        pipeline.withTraverseCommand( filterFactory.getEntityIdFilter( id ) );
+
+        this.collectorFilter = null;
+
+
+        return this;
+    }
+
+
+    @Override
+    public ReadPipelineBuilder getEntityViaCollection( final String collectionName, final Id entityId ) {
+
+        pipeline.withTraverseCommand( filterFactory.readGraphCollectionByIdFilter( collectionName, entityId ) );
+
+        setEntityLoaderFilter();
+
+        return this;
+    }
+
+
+    @Override
+    public ReadPipelineBuilder getCollection( final String collectionName ) {
+
+
+        pipeline.withTraverseCommand( filterFactory.readGraphCollectionCommand( collectionName ) );
+
+        setEntityLoaderFilter();
+
+        return this;
+    }
+
+
+    @Override
+    public ReadPipelineBuilder getCollectionWithQuery( final String collectionName, final String query, final int limit,
+                                                       final Optional<String> cursor ) {
+
+        //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+        collectorFilter = filterFactory.queryCollectionElasticSearchCollector( collectionName, query, cursor, limit );
+        return this;
+    }
+
+
+    @Override
+    public ReadPipelineBuilder getEntityViaConnection( final String connectionName, final Id entityId ) {
+
+        pipeline.withTraverseCommand( filterFactory.readGraphConnectionByIdFilter( connectionName, entityId ) );
+        setEntityLoaderFilter();
+
+        return this;
+    }
+
+
+    @Override
+    public ReadPipelineBuilder getConnection( final String connectionName ) {
+
+        pipeline.withTraverseCommand( filterFactory.readGraphConnectionCommand( connectionName ) );
+        setEntityLoaderFilter();
+
+        return this;
+    }
+
+
+    @Override
+    public ReadPipelineBuilder getConnection( final String connectionName, final String entityType ) {
+        pipeline.withTraverseCommand( filterFactory.readGraphConnectionCommand( connectionName, entityType ) );
+              setEntityLoaderFilter();
+
+        return this;
+    }
+
+    /**
+     *
+     * @param connectionName
+     * @param query
+     * @param limit
+     * @param cursor
+     * @return
+     */
+    @Override
+    public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String query, final int limit,
+                                                    final Optional<String> cursor ) {
+
+         //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+        collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, query, cursor, limit );
+
+        return this;
+    }
+
+
+    @Override
+    public ReadPipelineBuilder connectionWithQuery( final String connectionName, final String entityType,
+                                                    final String query, final int limit,
+                                                    final Optional<String> cursor ) {
+
+          //TODO, this should really be 2 a TraverseFilter with an entityLoad collector
+        collectorFilter = filterFactory.queryConnectionElasticSearchCollector( connectionName, entityType, query, cursor, limit );
+        return this;
+    }
+
+
+
+
+    @Override
+    public Observable<Results> execute() {
+        Preconditions.checkNotNull(collectorFilter, "You have not specified an operation that creates a collection filter.  This is required for loading results");
+
+        return pipeline.build( collectorFilter );
+    }
+
+    private void setEntityLoaderFilter(){
+        collectorFilter = filterFactory.entityLoadCollector();
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java
new file mode 100644
index 0000000..ba7c802
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/TraverseFilter.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read;
+
+
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Traverses edges in the graph.  Either by query or graph traversal.  Take an observable of ids, and emits
+ * an observable of ids
+ */
+public interface TraverseFilter extends Filter<Id> {}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
new file mode 100644
index 0000000..01ab21f
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/AbstractQueryElasticSearchCollectorFilter.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.AbstractFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.CollectorFilter;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ElasticSearchQueryExecutor;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchType;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+
+import rx.Observable;
+
+
+/**
+ * A command that will query and load elasticsearch
+ *
+ * On future iteration, this needs to be split into 2 commands 1 that loads the candidate results and validates the
+ * versions then another that will search and load
+ *
+ * TODO, split this into 3 seperate observables
+ *
+ * 1) An observable that emits candidate results 2) An observable that validates versions by uuid (for Traverse
+ * commands) 3) An observbale that emits Results as a collector (for final commands)
+ */
+public abstract class AbstractQueryElasticSearchCollectorFilter extends AbstractFilter<Results, Integer>
+    implements CollectorFilter<Results> {
+
+
+    protected final ApplicationEntityIndex entityIndex;
+    protected final Query query;
+    private int limit;
+
+
+    @Inject
+    protected AbstractQueryElasticSearchCollectorFilter( final ApplicationEntityIndex entityIndex, final Query query ) {
+        this.entityIndex = entityIndex;
+        this.query = query;
+    }
+
+
+    @Override
+    public Observable<Results> call( final Observable<Id> idObservable ) {
+
+        return idObservable.flatMap( id -> {
+
+            //TODO, refactor this logic to use Observables.  make this a TraverseFilter and load entities with the entity loader collector
+            final ResultsLoaderFactory resultsLoaderFactory = getResultsLoaderFactory( id );
+            final SearchEdge searchEdge = getSearchEdge( id );
+            final SearchTypes searchTypes = getSearchTypes();
+
+            final Iterable<Results> executor =
+                new ElasticSearchQueryExecutor( resultsLoaderFactory, entityIndex, applicationScope,
+                    searchEdge, searchTypes, query.withLimit( limit ) );
+
+            return Observable.from( executor );
+        } );
+    }
+
+
+    /**
+     * Get the search types
+     */
+    protected abstract SearchTypes getSearchTypes();
+
+    /**
+     * Get the search edge
+     */
+    protected abstract SearchEdge getSearchEdge(final Id incomingId);
+
+
+    /**
+     * Get the results loader factor
+     */
+    protected abstract ResultsLoaderFactory getResultsLoaderFactory( final Id incomingId );
+
+
+    @Override
+    protected Class<Integer> getCursorClass() {
+        return Integer.class;
+    }
+
+
+    @Override
+    public void setLimit( final int limit ) {
+        this.limit = limit;
+    }
+
+
+    /**
+     * Get an entiity ref from the Id.  TODO refactor this away
+     * @param id
+     * @return
+     */
+    protected EntityRef getRef(final Id id){
+        return new SimpleEntityRef( id.getType(), id.getUuid() );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
new file mode 100644
index 0000000..f0e8139
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryCollectionElasticSearchCollectorFilter.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createCollectionSearchEdge;
+
+
+/**
+ * Command for querying collections
+ */
+public class QueryCollectionElasticSearchCollectorFilter extends AbstractQueryElasticSearchCollectorFilter {
+
+    private final EntityCollectionManager entityCollectionManager;
+    private final ApplicationEntityIndex applicationEntityIndex;
+    private final String collectionName;
+
+
+    @Inject
+    protected QueryCollectionElasticSearchCollectorFilter( final EntityCollectionManager entityCollectionManager,
+                                                           final ApplicationEntityIndex applicationEntityIndex,
+                                                           @Assisted final String collectionName , @Assisted final Query query ) {
+        super( applicationEntityIndex, query );
+        this.entityCollectionManager = entityCollectionManager;
+        this.applicationEntityIndex = applicationEntityIndex;
+        this.collectionName = collectionName;
+    }
+
+
+    @Override
+    protected SearchTypes getSearchTypes() {
+        final SearchTypes types = SearchTypes.fromTypes( collectionName );
+
+        return types;
+    }
+
+
+    @Override
+    protected SearchEdge getSearchEdge( final Id incomingId ) {
+        final SearchEdge searchEdge = createCollectionSearchEdge( incomingId, collectionName );
+
+        return searchEdge;
+    }
+
+
+    @Override
+    protected ResultsLoaderFactory getResultsLoaderFactory( final Id id ) {
+
+        final EntityRef entityRef = getRef( id );
+        return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, applicationEntityIndex, entityRef,
+            collectionName );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
new file mode 100644
index 0000000..2b380df
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/QueryConnectionElasticSearchCollectorFilter.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch;
+
+
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ConnectionResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl.ResultsLoaderFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+
+import static org.apache.usergrid.corepersistence.util.CpNamingUtils.createConnectionSearchEdge;
+
+
+/**
+ * Command for querying connections
+ */
+public class QueryConnectionElasticSearchCollectorFilter extends AbstractQueryElasticSearchCollectorFilter {
+
+    private final EntityCollectionManager entityCollectionManager;
+    private final ApplicationEntityIndex applicationEntityIndex;
+    private final String connectionName;
+
+
+    @Inject
+    protected QueryConnectionElasticSearchCollectorFilter( final EntityCollectionManager entityCollectionManager,
+                                                           final ApplicationEntityIndex applicationEntityIndex,
+                                                           @Assisted final String connectionName,
+                                                           @Assisted final Query query ) {
+        super( applicationEntityIndex, query );
+        this.entityCollectionManager = entityCollectionManager;
+        this.applicationEntityIndex = applicationEntityIndex;
+        this.connectionName = connectionName;
+    }
+
+
+    @Override
+    protected SearchTypes getSearchTypes() {
+
+        final SearchTypes searchTypes = SearchTypes.fromNullableTypes( query.getEntityType() );
+
+        return searchTypes;
+    }
+
+
+    @Override
+    protected SearchEdge getSearchEdge( final Id id ) {
+        final SearchEdge searchEdge = createConnectionSearchEdge( id, connectionName );
+
+        return searchEdge;
+    }
+
+
+    @Override
+    protected ResultsLoaderFactory getResultsLoaderFactory( final Id id ) {
+        final EntityRef entityRef = getRef( id );
+        return new ConnectionResultsLoaderFactoryImpl( entityCollectionManager, applicationEntityIndex, entityRef,
+            connectionName );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java
new file mode 100644
index 0000000..cc96633
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionRefsVerifier.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class CollectionRefsVerifier extends VersionVerifier {
+
+
+
+    @Override
+    public Results getResults( final Collection<Id> ids ) {
+        List<EntityRef> refs = new ArrayList<EntityRef>(ids.size());
+        for ( Id id : ids ) {
+            refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+        }
+        return Results.fromRefList( refs );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..94c91d9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/CollectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public class CollectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+    private final EntityCollectionManager entityCollectionManager;
+    private final ApplicationEntityIndex applicationEntityIndex;
+
+
+    public CollectionResultsLoaderFactoryImpl( final EntityCollectionManager entityCollectionManager,
+        final ApplicationEntityIndex applicationEntityIndex ) {
+        this.entityCollectionManager = entityCollectionManager;
+        this.applicationEntityIndex = applicationEntityIndex;
+    }
+
+
+    @Override
+    public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope,
+                                    final Query.Level resultsLevel ) {
+
+        ResultsVerifier verifier;
+
+        if ( resultsLevel == Query.Level.REFS ) {
+            verifier = new CollectionRefsVerifier();
+        }
+        else if ( resultsLevel == Query.Level.IDS ) {
+            verifier = new IdsVerifier();
+        }
+        else {
+            verifier = new EntityVerifier( Query.MAX_LIMIT );
+        }
+
+        return new FilteringLoader( entityCollectionManager, applicationEntityIndex, verifier, applicationScope,
+            scope );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java
new file mode 100644
index 0000000..6b7bdde
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionRefsVerifier.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
+
+
+/**
+ * Verifier for creating connections
+ */
+public class ConnectionRefsVerifier extends VersionVerifier {
+
+
+    private final EntityRef ownerId;
+    private final String connectionType;
+
+
+    public ConnectionRefsVerifier( final EntityRef ownerId, final String connectionType ) {
+        this.ownerId = ownerId;
+        this.connectionType = connectionType;
+    }
+
+    @Override
+    public Results getResults( final Collection<Id> ids ) {
+        List<ConnectionRef> refs = new ArrayList<>();
+        for ( Id id : ids ) {
+            refs.add( new ConnectionRefImpl( ownerId, connectionType, ref(id.getType(), id.getUuid())  ));
+        }
+
+        return Results.fromConnections( refs );
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0523dc03/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..55b95a9
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/elasticsearch/impl/ConnectionResultsLoaderFactoryImpl.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.pipeline.read.elasticsearch.impl;
+
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Query;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.SearchEdge;
+
+
+/**
+ * Factory for creating results
+ */
+public class ConnectionResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+    private final EntityCollectionManager entityCollectionManager;
+    private final ApplicationEntityIndex applicationEntityIndex;
+    private final EntityRef ownerId;
+    private final String connectionType;
+
+
+    public ConnectionResultsLoaderFactoryImpl( final EntityCollectionManager entityCollectionManager,
+                                               final ApplicationEntityIndex applicationEntityIndex, final EntityRef ownerId,
+                                               final String connectionType ) {
+        this.entityCollectionManager = entityCollectionManager;
+        this.applicationEntityIndex = applicationEntityIndex;
+        this.ownerId = ownerId;
+        this.connectionType = connectionType;
+    }
+
+
+    @Override
+    public ResultsLoader getLoader( final ApplicationScope applicationScope, final SearchEdge scope,
+                                    final Query.Level resultsLevel ) {
+
+        ResultsVerifier verifier;
+
+        if ( resultsLevel == Query.Level.REFS ) {
+            verifier = new ConnectionRefsVerifier( ownerId, connectionType );
+        }
+        else if ( resultsLevel == Query.Level.IDS ) {
+            verifier = new ConnectionRefsVerifier( ownerId, connectionType );
+            ;
+        }
+        else {
+            verifier = new EntityVerifier( Query.MAX_LIMIT );
+        }
+
+        return new FilteringLoader( entityCollectionManager, applicationEntityIndex, verifier, applicationScope, scope );
+    }
+}