You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/01 17:27:44 UTC

[1/5] incubator-usergrid git commit: Initial commit. WIP overwrite

Repository: incubator-usergrid
Updated Branches:
  refs/heads/USERGRID-528 [created] cee76da72


Initial commit. WIP overwrite


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fe1fa85c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fe1fa85c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fe1fa85c

Branch: refs/heads/USERGRID-528
Commit: fe1fa85c36b5eb772c5ff03be816d71e1522793d
Parents: 61c5f9c
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 19 09:47:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 19 09:47:43 2015 -0600

----------------------------------------------------------------------
 .../io/execution/CommandExecution.java          | 37 +++++++++++
 .../corepersistence/io/read/Command.java        | 38 ++++++++++++
 .../corepersistence/io/read/CommandBuilder.java | 65 ++++++++++++++++++++
 .../io/reduce/StreamReducer.java                | 38 ++++++++++++
 4 files changed, 178 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java
new file mode 100644
index 0000000..0aaa764
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.execution;
+
+
+import org.apache.usergrid.corepersistence.io.read.CommandBuilder;
+import org.apache.usergrid.corepersistence.io.reduce.StreamReducer;
+
+
+public interface CommandExecution {
+
+    /**
+     * Invoke the commands, then feed them to the reducer to return a final output
+     * @param builder
+     * @param reducer
+     * @param <T>
+     * @return
+     */
+    public <T> T invoke(CommandBuilder builder, StreamReducer<T> reducer);
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
new file mode 100644
index 0000000..4e3b0fc
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+public interface Command<T> {
+
+
+    /**
+     * Process our input stream
+     * @param input
+     * @return
+     */
+    public Observable<T> process( Observable<T> input );
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
new file mode 100644
index 0000000..119fc0e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.usergrid.corepersistence.io.reduce.StreamReducer;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+import rx.functions.Func1;
+
+
+/**
+ * The builder to hold the list of traversal commands
+ */
+public class CommandBuilder {
+
+    private final Id root;
+    private final List<Command<Id>> commandList;
+
+
+    public CommandBuilder( final Id root ) {this.root = root;
+        commandList = new ArrayList<>(  );
+    }
+
+    public void addIntermediateCommand(final Command<Id> command){
+      commandList.add( command );
+    }
+
+
+    public <T> void addFinalCommand( final Command<T> command, final StreamReducer<T> reducer ) {
+
+        Observable.just("foo").flatMap( new Func1<String, Observable<?>>() {
+            @Override
+            public Observable<?> call( final String s ) {
+                return null;
+            }
+        };
+    }
+
+
+    public List<Command<Id>> getCommands(){
+        return  commandList;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java
new file mode 100644
index 0000000..ae84bd8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.reduce;
+
+
+import org.apache.usergrid.corepersistence.io.read.Command;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+public interface StreamReducer<T> {
+
+    /**
+     * Reduce the result of our stream into a list of nodes
+     *
+     * @param candidates The candidate ids for loading
+     * @return
+     */
+    public T reduce( Observable<Id> candidates );
+}


[2/5] incubator-usergrid git commit: Merge branch 'USERGRID-405' into USERGRID-494

Posted by to...@apache.org.
Merge branch 'USERGRID-405' into USERGRID-494


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c59f092
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c59f092
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c59f092

Branch: refs/heads/USERGRID-528
Commit: 4c59f09222c00183e994c49ad3f3ed96938bcd22
Parents: fe1fa85 a40fb8d
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 20 15:54:53 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 20 15:54:53 2015 -0600

----------------------------------------------------------------------
 .../main/dist/init_instance/init_rest_server.sh |   6 +-
 stack/core/pom.xml                              |  54 +-
 .../batch/service/SchedulerServiceImpl.java     |  17 +-
 .../AllApplicationsObservable.java              |  39 ++
 .../usergrid/corepersistence/CoreModule.java    |  88 ++-
 .../corepersistence/CpEntityManager.java        |  43 +-
 .../corepersistence/CpEntityManagerFactory.java | 160 ++---
 .../corepersistence/CpManagerCache.java         |  57 +-
 .../corepersistence/CpRelationManager.java      |  43 +-
 .../usergrid/corepersistence/CpSetup.java       |   6 +-
 .../usergrid/corepersistence/CpWalker.java      |  81 +--
 .../usergrid/corepersistence/ManagerCache.java  |  13 +-
 .../events/EntityDeletedHandler.java            |   3 +-
 .../events/EntityVersionDeletedHandler.java     |  76 +--
 .../migration/CoreDataVersions.java             |  43 ++
 .../migration/CoreMigration.java                |  53 ++
 .../migration/CoreMigrationPlugin.java          |  64 ++
 .../migration/EntityDataMigration.java          | 147 ----
 .../migration/EntityTypeMappingMigration.java   |  69 +-
 .../migration/GraphShardVersionMigration.java   | 152 -----
 .../migration/MigrationModuleVersionPlugin.java | 133 ++++
 .../migration/MigrationSystemVersions.java      |  43 ++
 .../corepersistence/migration/Versions.java     |  14 -
 .../results/FilteringLoader.java                |   3 +-
 .../rx/AllEntitiesInSystemObservable.java       | 101 ---
 .../rx/ApplicationObservable.java               | 128 ----
 .../rx/EdgesFromSourceObservable.java           |  63 --
 .../rx/EdgesToTargetObservable.java             |  63 --
 .../corepersistence/rx/TargetIdObservable.java  |  66 --
 .../rx/impl/AbstractGraphVisitorImpl.java       | 107 +++
 .../rx/impl/AllApplicationsObservableImpl.java  | 142 ++++
 .../rx/impl/AllEntitiesInSystemImpl.java        |  62 ++
 .../rx/impl/AllNodesInGraphImpl.java            |  55 ++
 .../corepersistence/util/CpEntityMapUtils.java  |  19 +-
 .../corepersistence/util/CpNamingUtils.java     |   2 +
 .../usergrid/persistence/EntityManager.java     |  19 +-
 .../persistence/EntityManagerFactory.java       |  22 -
 .../persistence/ObservableIterator.java         |   1 +
 .../cassandra/EntityManagerFactoryImpl.java     |  28 -
 .../cassandra/EntityManagerImpl.java            |  20 +-
 .../org/apache/usergrid/CoreApplication.java    |  24 +-
 .../java/org/apache/usergrid/CoreITSetup.java   |   6 +
 .../org/apache/usergrid/CoreITSetupImpl.java    |  27 +-
 .../org/apache/usergrid/TestEntityIndex.java    |  27 +
 .../corepersistence/StaleIndexCleanupTest.java  |  87 +--
 .../migration/EntityDataMigrationIT.java        | 262 -------
 .../migration/EntityTypeMappingMigrationIT.java | 183 ++---
 .../migration/GraphShardVersionMigrationIT.java | 226 ------
 .../MigrationModuleVersionPluginTest.java       | 259 +++++++
 .../migration/MigrationTestRule.java            |  99 ---
 .../migration/TestProgressObserver.java         |  71 --
 .../rx/AllEntitiesInSystemObservableIT.java     |  41 +-
 .../rx/ApplicationObservableTestIT.java         |  14 +-
 .../rx/EdgesFromSourceObservableIT.java         |  10 +-
 .../rx/EdgesToTargetObservableIT.java           |   8 +-
 .../rx/TargetIdObservableTestIT.java            |   7 +-
 .../usergrid/persistence/CollectionIT.java      |  80 +--
 .../usergrid/persistence/CountingMutatorIT.java |   4 +-
 .../persistence/EntityConnectionsIT.java        |  14 +-
 .../usergrid/persistence/EntityManagerIT.java   |  16 +-
 .../org/apache/usergrid/persistence/GeoIT.java  |  27 +-
 .../persistence/GeoQueryBooleanTest.java        |   4 +-
 .../apache/usergrid/persistence/IndexIT.java    |  14 +-
 .../usergrid/persistence/LargeEntityIT.java     |   3 +-
 .../usergrid/persistence/PathQueryIT.java       |   6 +-
 .../PerformanceEntityRebuildIndexTest.java      |  39 +-
 .../usergrid/persistence/PermissionsIT.java     |   4 +-
 .../cassandra/EntityManagerFactoryImplIT.java   |   4 +-
 .../persistence/query/CollectionIoHelper.java   |   2 +-
 .../persistence/query/ConnectionHelper.java     |   2 +-
 .../query/IntersectionTransitivePagingIT.java   |   2 +-
 .../query/IntersectionUnionPagingIT.java        |   2 +-
 .../persistence/query/IteratingQueryIT.java     |  34 +-
 .../resources/usergrid-custom-test.properties   |   1 +
 stack/corepersistence/collection/pom.xml        |  14 +-
 .../collection/EntityCollectionManager.java     |   6 -
 .../EntityCollectionManagerFactory.java         |  27 +-
 .../collection/EntityCollectionManagerSync.java |  49 --
 .../persistence/collection/MvccEntity.java      |   5 +-
 .../persistence/collection/MvccLogEntry.java    |   1 +
 .../cache/CachedEntityCollectionManager.java    |   7 -
 .../collection/event/EntityVersionDeleted.java  |   3 +-
 .../collection/guice/CollectionModule.java      |  45 +-
 .../persistence/collection/guice/Write.java     |  17 -
 .../collection/guice/WriteUpdate.java           |  17 -
 .../EntityCollectionManagerFactoryImpl.java     |  71 +-
 .../impl/EntityCollectionManagerImpl.java       | 129 +---
 .../impl/EntityCollectionManagerSyncImpl.java   |  71 --
 .../collection/impl/EntityDeletedTask.java      |  28 +-
 .../impl/EntityVersionCleanupTask.java          | 183 ++---
 .../impl/EntityVersionCreatedTask.java          |  26 +-
 .../mvcc/MvccEntitySerializationStrategy.java   | 102 ---
 .../mvcc/stage/delete/MarkCommit.java           |   2 +-
 .../mvcc/stage/write/WriteCommit.java           |  11 +-
 .../collection/mvcc/stage/write/WriteStart.java |   6 +-
 .../mvcc/stage/write/WriteUniqueVerify.java     |   8 +-
 .../collection/serialization/EntityRepair.java  |  38 --
 .../MvccEntitySerializationStrategy.java        | 119 ++++
 .../serialization/OptimisticUpdate.java         |  23 -
 .../UniqueValueSerializationStrategy.java       |  46 +-
 .../serialization/UniqueValueSet.java           |   4 +
 .../impl/CollectionDataVersions.java            |  44 ++
 .../serialization/impl/EntityRepairImpl.java    | 149 ----
 .../impl/EntityVersionSerializer.java           |   2 -
 .../serialization/impl/FieldSerializer.java     | 118 ----
 .../MvccEntitySerializationStrategyImpl.java    | 112 ++-
 ...vccEntitySerializationStrategyProxyImpl.java | 151 ++--
 .../MvccEntitySerializationStrategyV1Impl.java  |   9 +-
 .../MvccEntitySerializationStrategyV2Impl.java  |   8 +-
 .../MvccEntitySerializationStrategyV3Impl.java  | 587 ++++++++++++++++
 .../serialization/impl/SerializationModule.java |  85 ++-
 .../serialization/impl/UniqueFieldEntry.java    |  61 ++
 .../impl/UniqueFieldEntrySerializer.java        | 141 ++++
 .../impl/UniqueFieldRowKeySerializer.java       | 122 ++++
 .../UniqueValueSerializationStrategyImpl.java   | 248 +++++--
 .../impl/migration/CollectionMigration.java     |  53 ++
 .../migration/CollectionMigrationPlugin.java    |  68 ++
 .../impl/migration/EntityIdScope.java           |  49 ++
 .../migration/MvccEntityDataMigrationImpl.java  | 260 +++++++
 .../collection/util/EntityUtils.java            |  71 --
 .../collection/EntityCollectionManagerIT.java   | 104 +--
 .../EntityCollectionManagerSyncIT.java          | 193 ------
 .../collection/guice/TestCollectionModule.java  |  22 +-
 .../impl/EntityVersionCleanupTaskTest.java      | 680 ++++++++-----------
 .../impl/EntityVersionCreatedTaskTest.java      |   2 -
 .../mvcc/stage/AbstractEntityStageTest.java     |   2 +-
 .../mvcc/stage/AbstractMvccEntityStageTest.java |   2 +-
 .../mvcc/stage/TestEntityGenerator.java         |   2 +-
 .../mvcc/stage/delete/MarkCommitTest.java       |   2 +-
 .../mvcc/stage/write/FieldSerializerTest.java   |  52 --
 .../write/UniqueFieldRowKeySerializerTest.java  |  52 ++
 ...niqueValueSerializationStrategyImplTest.java | 149 +++-
 .../mvcc/stage/write/WriteCommitTest.java       |   2 +-
 .../mvcc/stage/write/WriteStartTest.java        |   6 +-
 .../persistence/collection/rx/ParallelTest.java |  10 +-
 .../serialization/EntityRepairImplTest.java     | 147 ----
 .../impl/LogEntryIteratorTest.java              |   3 +-
 ...MvccEntitySerializationStrategyImplTest.java | 121 +---
 ...cEntitySerializationStrategyProxyV1Test.java |  85 ---
 ...ntitySerializationStrategyProxyV1_3Test.java |  83 +++
 ...cEntitySerializationStrategyProxyV2Test.java |  83 ---
 ...ntitySerializationStrategyProxyV2_3Test.java |  84 +++
 ...ccEntitySerializationStrategyV1ImplTest.java |  80 ++-
 ...ccEntitySerializationStrategyV2ImplTest.java |  75 +-
 .../MvccEntitySerializationStrategyV2Test.java  |   7 +-
 ...ccEntitySerializationStrategyV3ImplTest.java |  74 ++
 .../impl/SerializationComparison.java           |   4 +-
 .../impl/UniqueFieldEntrySerializerTest.java    | 108 +++
 ...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 215 ++++++
 .../MvccEntityDataMigrationV1ToV3ImplTest.java  | 105 +++
 .../MvccEntityDataMigrationV2ToV3ImplTest.java  |  99 +++
 .../collection/util/InvalidEntityGenerator.java |   1 +
 .../collection/util/LogEntryMock.java           |  39 +-
 .../collection/util/UniqueValueEntryMock.java   | 161 +++++
 .../collection/util/VersionGenerator.java       |  55 ++
 stack/corepersistence/common/pom.xml            |  15 +-
 .../usergrid/persistence/core/CPManager.java    |  28 +
 .../astyanax/DynamicCompositeParserImpl.java    |  81 +++
 .../core/astyanax/FieldBufferBuilder.java       |  15 +
 .../core/astyanax/FieldBufferParser.java        |  13 +
 .../astyanax/MultiKeyColumnNameIterator.java    |   4 +-
 .../core/astyanax/MultiRowColumnIterator.java   |  12 +-
 .../persistence/core/guice/CommonModule.java    |  17 +-
 .../persistence/core/guice/CurrentImpl.java     |  42 --
 .../persistence/core/guice/PreviousImpl.java    |  42 --
 .../core/hystrix/HystrixCassandra.java          |  94 ---
 .../migration/data/AbstractMigrationPlugin.java | 128 ++++
 .../core/migration/data/DataMigration.java      |  97 +--
 .../migration/data/DataMigrationManager.java    |  18 +-
 .../data/DataMigrationManagerImpl.java          | 234 ++++---
 .../migration/data/MigrationDataProvider.java   |  43 ++
 .../core/migration/data/MigrationInfoCache.java |  52 ++
 .../migration/data/MigrationInfoCacheImpl.java  |  84 +++
 .../data/MigrationInfoSerialization.java        |  28 +-
 .../data/MigrationInfoSerializationImpl.java    |  67 +-
 .../core/migration/data/MigrationPlugin.java    |  56 ++
 .../migration/data/MigrationRelationship.java   | 100 +++
 .../core/migration/data/PluginPhase.java        |  42 ++
 .../core/migration/data/ProgressObserver.java   |  63 ++
 .../core/migration/data/VersionedData.java      |  38 ++
 .../migration/data/VersionedMigrationSet.java   | 153 +++++
 .../core/scope/ApplicationScope.java            |   1 +
 .../persistence/core/task/TaskExecutor.java     |   4 +-
 .../MultiKeyColumnNameIteratorTest.java         | 187 +++--
 .../astyanax/MultiRowColumnIteratorTest.java    |  50 +-
 .../core/guice/DataMigrationResetRule.java      |  88 +++
 .../core/guice/MaxMigrationModule.java          |  39 --
 .../core/guice/MaxMigrationVersion.java         |  40 --
 .../core/guice/MigrationManagerRule.java        |   7 +
 .../core/guice/TestCommonModule.java            |   1 +
 .../persistence/core/guice/TestModule.java      |   5 +-
 .../data/DataMigrationManagerImplTest.java      | 350 ++++++----
 .../data/MigrationInfoSerializationTest.java    |  32 +-
 .../data/TestMigrationDataProvider.java         |  61 ++
 .../migration/data/TestProgressObserver.java    |  89 +++
 .../data/VersionedMigrationSetTest.java         | 198 ++++++
 .../persistence/core/util/IdGenerator.java      |  51 ++
 .../persistence/graph/GraphManager.java         |   3 +-
 .../persistence/graph/GraphManagerFactory.java  |   2 +
 .../persistence/graph/guice/GraphModule.java    | 103 ++-
 .../graph/impl/GraphManagerImpl.java            |  38 +-
 .../graph/impl/stage/EdgeDeleteRepairImpl.java  |   9 +-
 .../graph/impl/stage/EdgeMetaRepairImpl.java    |  18 +-
 .../impl/stage/NodeDeleteListenerImpl.java      |  18 +-
 .../EdgeMetadataSerialization.java              |   3 +-
 .../graph/serialization/EdgesObservable.java    |  33 +
 .../graph/serialization/TargetIdObservable.java |  38 ++
 .../EdgeMetadataSerializationProxyImpl.java     | 161 +++--
 .../impl/EdgeMetadataSerializationV1Impl.java   |   6 +
 .../impl/EdgeMetadataSerializationV2Impl.java   |   6 +
 .../serialization/impl/EdgesObservableImpl.java |  80 +++
 .../serialization/impl/GraphDataVersions.java   |  43 ++
 .../impl/GraphManagerFactoryImpl.java           |  95 +++
 .../impl/NodeSerializationImpl.java             |  36 +-
 .../impl/TargetIdObservableImpl.java            |  72 ++
 .../impl/migration/EdgeDataMigrationImpl.java   | 138 ++++
 .../impl/migration/GraphMigration.java          |  53 ++
 .../impl/migration/GraphMigrationPlugin.java    |  69 ++
 .../serialization/impl/migration/GraphNode.java |  39 ++
 .../shard/count/NodeShardApproximationImpl.java |   4 +-
 .../NodeShardCounterSerializationImpl.java      |  25 +-
 .../shard/impl/NodeShardAllocationImpl.java     |  16 +-
 .../shard/impl/ShardGroupCompactionImpl.java    |  64 +-
 .../persistence/graph/GraphManagerIT.java       |  38 +-
 .../persistence/graph/GraphManagerLoadTest.java |  13 +-
 .../graph/GraphManagerShardConsistencyIT.java   |  12 +-
 .../graph/GraphManagerShardingIT.java           |  13 +-
 .../graph/GraphManagerStressTest.java           |  11 +-
 .../usergrid/persistence/graph/SimpleTest.java  |  12 +-
 .../graph/guice/TestGraphModule.java            |  27 +-
 .../graph/impl/EdgeDeleteListenerTest.java      |  11 +-
 .../graph/impl/NodeDeleteListenerTest.java      |   9 +-
 .../graph/impl/stage/EdgeDeleteRepairTest.java  |   7 +-
 .../graph/impl/stage/EdgeMetaRepairTest.java    |  29 +-
 .../EdgeMetaDataSerializationProxyV1Test.java   |  14 +-
 .../EdgeMetaDataSerializationProxyV2Test.java   |  21 +-
 .../EdgeMetaDataSerializationV1Test.java        |   7 +-
 .../EdgeMetaDataSerializationV2Test.java        |   7 +-
 .../EdgeMetadataSerializationTest.java          |  39 +-
 .../EdgeSerializationChopTest.java              |   7 +-
 .../serialization/EdgeSerializationTest.java    |  21 +-
 .../serialization/NodeSerializationTest.java    |  15 +-
 .../migration/EdgeDataMigrationImplTest.java    | 177 +++++
 .../impl/shard/EdgeShardSerializationTest.java  |   7 +-
 .../impl/shard/NodeShardAllocationTest.java     |  27 +-
 .../impl/shard/NodeShardCacheTest.java          |   7 +-
 .../impl/shard/ShardGroupCompactionTest.java    |   7 +-
 .../shard/count/NodeShardApproximationTest.java |   9 +-
 .../NodeShardCounterSerializationTest.java      |   6 +-
 .../shard/impl/ShardEntryGroupIteratorTest.java |  15 +-
 ...rceDirectedEdgeDescendingComparatorTest.java |  23 +-
 ...getDirectedEdgeDescendingComparatorTest.java |  23 +-
 .../graph/test/util/EdgeTestUtils.java          |  29 +-
 .../usergrid/persistence/map/MapManager.java    |  25 +-
 .../persistence/map/guice/MapModule.java        |   8 +-
 .../map/impl/MapManagerFactoryImpl.java         |  29 +-
 .../persistence/map/guice/TestMapModule.java    |   3 +-
 stack/corepersistence/model/pom.xml             |   1 -
 .../persistence/model/entity/Entity.java        |  23 +-
 .../persistence/model/entity/EntityMap.java     |  66 ++
 .../model/entity/EntityToMapConverter.java      | 133 ++++
 .../model/entity/MapToEntityConverter.java      | 203 ++++++
 .../persistence/model/entity/SimpleId.java      |   1 -
 .../persistence/model/field/AbstractField.java  |   8 +
 .../persistence/model/field/ListField.java      |   2 +-
 .../persistence/model/field/StringField.java    |  20 +
 .../persistence/model/util/EntityUtils.java     |  72 ++
 stack/corepersistence/pom.xml                   |   8 +-
 stack/corepersistence/queryindex/pom.xml        |  63 +-
 .../persistence/index/AliasedEntityIndex.java   |   7 +-
 .../index/ApplicationEntityIndex.java           |  50 ++
 .../usergrid/persistence/index/EntityIndex.java |  50 +-
 .../persistence/index/EntityIndexBatch.java     |   5 +
 .../persistence/index/EntityIndexFactory.java   |   2 +-
 .../persistence/index/IndexIdentifier.java      |  26 +-
 .../usergrid/persistence/index/SearchType.java  |  49 ++
 .../usergrid/persistence/index/SearchTypes.java |  14 +-
 .../persistence/index/guice/IndexModule.java    |   5 +-
 .../impl/EsApplicationEntityIndexImpl.java      | 371 ++++++++++
 .../index/impl/EsEntityIndexBatchImpl.java      |  80 ++-
 .../index/impl/EsEntityIndexFactoryImpl.java    |  27 +-
 .../index/impl/EsEntityIndexImpl.java           | 456 +++----------
 .../persistence/index/impl/EsQueryVistor.java   |  88 +--
 .../persistence/index/impl/IndexingUtils.java   | 140 ++--
 .../persistence/index/query/EntityResults.java  | 108 ---
 .../usergrid/persistence/index/query/Query.java |   6 +-
 .../persistence/index/query/Results.java        | 148 ----
 .../persistence/index/utils/ListUtils.java      |   6 +-
 .../persistence/index/guice/IndexTestFig.java   |  62 ++
 .../index/guice/TestIndexModule.java            |  11 +-
 .../index/impl/CorePerformanceIT.java           | 339 ---------
 .../impl/EntityConnectionIndexImplTest.java     | 306 ---------
 .../index/impl/EntityIndexMapUtils.java         |  52 +-
 .../persistence/index/impl/EntityIndexTest.java | 199 ++----
 .../index/impl/IndexLoadTestsIT.java            | 378 +++++++++++
 .../queue/guice/TestQueueModule.java            |   3 +-
 stack/pom.xml                                   |  16 +-
 stack/rest/pom.xml                              |   1 +
 .../usergrid/rest/AbstractContextResource.java  | 102 +--
 .../org/apache/usergrid/rest/IndexResource.java |   5 +-
 .../apache/usergrid/rest/MigrateResource.java   |  94 ++-
 .../rest/test/RefreshIndexResource.java         |  17 +-
 .../resources/usergrid-custom-test.properties   |   2 +
 stack/services/pom.xml                          |   1 +
 .../management/importer/ImportServiceImpl.java  |  34 +-
 .../impl/ApplicationQueueManagerImpl.java       | 195 +++---
 .../org/apache/usergrid/ServiceApplication.java |  18 +-
 .../apache/usergrid/management/EmailFlowIT.java |   6 +-
 .../usergrid/management/OrganizationIT.java     |  10 +-
 .../org/apache/usergrid/management/RoleIT.java  |   4 +-
 .../cassandra/ManagementServiceIT.java          |  22 +-
 .../management/export/ExportServiceIT.java      |  12 +-
 .../management/importer/ImportCollectionIT.java |  15 +-
 .../usergrid/services/ConnectionsServiceIT.java |  10 +-
 .../usergrid/services/GroupServiceIT.java       |   6 +-
 .../usergrid/services/RolesServiceIT.java       |   2 +-
 .../usergrid/services/UsersServiceIT.java       |   2 +-
 .../AbstractServiceNotificationIT.java          |   2 +-
 .../apns/NotificationsServiceIT.java            |  40 +-
 .../gcm/NotificationsServiceIT.java             |   4 +-
 .../resources/usergrid-custom-test.properties   |   2 +
 .../setup/ConcurrentProcessSingleton.java       |  16 +-
 322 files changed, 11294 insertions(+), 7800 deletions(-)
----------------------------------------------------------------------



[3/5] incubator-usergrid git commit: WIP and a mess, squash later.

Posted by to...@apache.org.
WIP and a mess, squash later.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c4f65484
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c4f65484
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c4f65484

Branch: refs/heads/USERGRID-528
Commit: c4f654847b8041a94176996feffa4060dd638819
Parents: 4c59f09
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 20 19:10:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 20 19:10:58 2015 -0600

----------------------------------------------------------------------
 .../corepersistence/io/read/CommandBuilder.java |  36 +++---
 .../io/read/EntityIndexCommands.java            | 119 +++++++++++++++++++
 .../corepersistence/io/state/CursorCache.java   |  34 ++++++
 .../corepersistence/rx/impl/CollectUntil.java   |  60 ++++++++++
 .../java/org/apache/usergrid/TempExample.java   |  74 ++++++++++++
 .../rx/impl/CollectUntilTest.java               |  62 ++++++++++
 6 files changed, 368 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
index 119fc0e..828cbc9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.usergrid.corepersistence.io.reduce.StreamReducer;
+import org.apache.usergrid.corepersistence.io.state.CursorCache;
 import org.apache.usergrid.persistence.model.entity.Id;
 
 import rx.Observable;
@@ -35,31 +36,32 @@ import rx.functions.Func1;
  */
 public class CommandBuilder {
 
-    private final Id root;
-    private final List<Command<Id>> commandList;
 
+    private CursorCache cache;
+    private final Observable<Id> pathObservable;
 
-    public CommandBuilder( final Id root ) {this.root = root;
-        commandList = new ArrayList<>(  );
-    }
 
-    public void addIntermediateCommand(final Command<Id> command){
-      commandList.add( command );
+    public CommandBuilder( final Id root ) {
+       pathObservable = Observable.just( root );
     }
 
 
-    public <T> void addFinalCommand( final Command<T> command, final StreamReducer<T> reducer ) {
-
-        Observable.just("foo").flatMap( new Func1<String, Observable<?>>() {
-            @Override
-            public Observable<?> call( final String s ) {
-                return null;
-            }
-        };
+    /**
+     * Set our cache
+     * @param cache
+     */
+    public void setCache(final CursorCache cache){
+        this.cache = cache;
     }
 
 
-    public List<Command<Id>> getCommands(){
-        return  commandList;
+    /**
+     * Returns the observable that contains the current traversal operations
+     * @return
+     */
+    public Observable<Id> getPathObservable(){
+        return pathObservable;
     }
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
new file mode 100644
index 0000000..d6f4e93
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+import rx.functions.Action2;
+import rx.functions.Func1;
+
+
+public class EntityIndexCommands {
+
+
+//    /**
+//     * Perform a search of all the entities, and then return the observable of search results
+//     * @param index
+//     * @param edgeType
+//     * @param types
+//     * @param query
+//     * @return
+//     */
+//    public static Func1<Id, SearchResults> searchEntities(final ApplicationEntityIndex index,final String edgeType, final SearchTypes types, final String query  ){
+//
+//        return nodeId -> {
+//
+//        }
+//    }
+    /**
+     * Construct an indexScope from the input id type
+     * @param type
+     * @return
+     */
+   public static Func1<Id, IndexScope> createSearchScope(final String type){
+       return id -> new IndexScopeImpl( id, type );
+   }
+    /**
+     * Get our candidate results
+     * @param index
+     * @param types The types to return
+     * @param query
+     * @return
+     */
+    public static Func1<IndexScope, CandidateResults> getCandidates(final ApplicationEntityIndex index, final SearchTypes types, final String query){
+        return indexScope -> index.search( indexScope, types, Query.fromQLNullSafe( query ) );
+    }
+
+
+    /**
+     * Flattens candidate results into a single stream of a result
+     * @return
+     */
+    public static Func1<CandidateResults, Observable<CandidateResult>> flattenCandidates(){
+        return (CandidateResults candidateResults) -> Observable.from( candidateResults );
+    }
+
+
+    public static Action2<SearchResults, EntitySet> collectSet(){
+        return (searchResults, entitySet) -> {
+          searchResults.addEntities( entitySet.entities );
+        };
+    }
+
+
+
+    public static class SearchResults{
+        private final List<Entity> entities;
+        private String cursor;
+
+
+        public SearchResults(final int maxSize) {entities = new ArrayList<>(maxSize);}
+
+        public void addEntities(final Collection<Entity> entities){
+            this.entities.addAll( entities );
+
+        }
+        public void setCursor(final String cursor){
+            this.cursor = cursor;
+        }
+    }
+
+    public static class EntitySet{
+        private final List<Entity> entities;
+
+
+        public EntitySet( final List<Entity> entities ) {this.entities = entities;}
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java
new file mode 100644
index 0000000..8a5ca49
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.state;
+
+
+/**
+ * A class that represents a cursor cache
+ */
+public class CursorCache {
+
+    public CursorCache(){
+
+    }
+
+}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
new file mode 100644
index 0000000..a2cb754
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import rx.Observable;
+import rx.functions.Action2;
+import rx.functions.Func0;
+import rx.functions.Func1;
+import rx.functions.Func2;
+import rx.internal.operators.OperatorScan;
+
+
+/**
+ * An operation for performing a collect until the predicate returns true
+ */
+public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
+
+    final Func0<R> stateFactory;
+    final Action2<R, ? super T> collector;
+    final Func1<R, Boolean> predicate;
+
+
+    public CollectUntil( final Func0<R> stateFactory, final Action2<R, ? super T> collector,
+                          final Func1<R, Boolean> predicate ) {
+        this.stateFactory = stateFactory;
+        this.collector = collector;
+        this.predicate = predicate;
+    }
+
+
+    @Override
+    public Observable<R> call( final Observable<T> tObservable ) {
+        Func2<R, T, R> accumulator = ( state, value ) -> {
+            collector.call( state, value );
+            return state;
+        };
+
+
+        return tObservable.lift( new OperatorScan<>( stateFactory, accumulator ) ).takeUntil( predicate );
+    }
+}
+

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/test/java/org/apache/usergrid/TempExample.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/TempExample.java b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
new file mode 100644
index 0000000..1db450d
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid;
+
+
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.createSearchScope;
+import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.getCandidates;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+
+
+public class TempExample {
+
+    //set our root observable
+
+
+    public static void main(String[] args) {
+
+        final Id rootId = createId( "thing" );
+
+        final ApplicationEntityIndex index = null;
+
+
+        final SearchTypes searchType = SearchTypes.fromTypes( "test" );
+
+        final String query = "select * ";
+
+        final Observable<CandidateResults> observable = Observable.just( rootId ).map( createSearchScope( "type" ) ).map(getCandidates(index, searchType, query));
+
+
+        observable.doOnNext( a -> System.out.println( a) ).toBlocking().last();
+    }
+
+
+    private static final class ResultsCollector{
+
+        /**
+         * Add the candidates to our collection
+         * @param results
+         */
+        public void addCandidates(final CandidateResults results ){
+
+            //TODO, collect the results, removing groups
+            Observable.from( results ).groupBy( candidate -> candidate.getId() ).collect(  )
+        }
+
+    }
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
new file mode 100644
index 0000000..ce12429
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import org.junit.Test;
+
+import rx.Observable;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class CollectUntilTest {
+
+    @Test
+    public void testCollectUntil() {
+
+        final CollectUntil<Integer, CountCollector> collectUntil =
+            new CollectUntil<>(
+                () -> new CountCollector(),
+                ( collector, value ) -> collector.mark(),
+                collector -> collector.isFull() );
+
+
+        final CountCollector collector = Observable.range( 0, 200 ).compose( collectUntil ).toBlocking().last();
+
+        assertEquals( 100, collector.count );
+    }
+
+
+    private static final class CountCollector {
+
+        private int count;
+
+
+        public void mark() {
+            count++;
+        }
+
+
+        public boolean isFull() {
+            return count == 100;
+        }
+    }
+}


[5/5] incubator-usergrid git commit: WIP, overwrite

Posted by to...@apache.org.
WIP, overwrite


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cee76da7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cee76da7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cee76da7

Branch: refs/heads/USERGRID-528
Commit: cee76da72c3343d562867988f76f264204d54b59
Parents: 61aa979
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 24 14:15:49 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 24 14:15:49 2015 -0600

----------------------------------------------------------------------
 .../io/read/EntityIndexCommand.java             | 404 ++++++++++++-------
 .../corepersistence/rx/impl/CollectUntil.java   |   2 +-
 2 files changed, 270 insertions(+), 136 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cee76da7/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
index a8fa221..464dd45 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
@@ -22,8 +22,9 @@ package org.apache.usergrid.corepersistence.io.read;
 
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.collection.CollectionScope;
@@ -32,8 +33,8 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory
 import org.apache.usergrid.persistence.collection.EntitySet;
 import org.apache.usergrid.persistence.collection.MvccEntity;
 import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
 import org.apache.usergrid.persistence.index.query.CandidateResult;
@@ -52,32 +53,34 @@ import rx.functions.Func0;
 import rx.functions.Func1;
 
 
+/**
+ * Performs a search of the given type along the specified edgeName.  It then loads and validates results,
+ * and returns an Observable of SearchResults
+ */
 @Singleton
 public class EntityIndexCommand implements Command<Id, EntityIndexCommand.SearchResults> {
 
     private final Id applicationId;
-    private final ApplicationScope applicationScope;
     private final ApplicationEntityIndex index;
     private final SearchTypes types;
     private final String query;
     private final int resultSetSize;
-    private final String scopeType;
+    private final String edgeName;
     private final EntityCollectionManagerFactory entityCollectionManagerFactory;
 
 
     @Inject
-    public EntityIndexCommand( final Id applicationId, final ApplicationScope applicationScope,
-                               final ApplicationEntityIndex index, final SearchTypes types, final String query,
-                               final int resultSetSize, final String scopeType,
+    public EntityIndexCommand( final Id applicationId, final ApplicationEntityIndex index, final SearchTypes types,
+                               final String query, final int resultSetSize, final String edgeName,
                                final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
         this.applicationId = applicationId;
-        this.applicationScope = applicationScope;
+
 
         this.index = index;
         this.types = types;
         this.query = query;
         this.resultSetSize = resultSetSize;
-        this.scopeType = scopeType;
+        this.edgeName = edgeName;
         this.entityCollectionManagerFactory = entityCollectionManagerFactory;
     }
 
@@ -85,150 +88,216 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
     @Override
     public Observable<EntityIndexCommand.SearchResults> call( final Observable<Id> idObservable ) {
 
-        //create our observable of candidate search results
-        final Observable<CandidateResults> candidateResults = idObservable
-            .flatMap( id -> Observable.create( new ElasticSearchObservable( initialSearch( id ), nextPage( id ) ) ) );
-
-        final Observable<CandidateResult> candidateObservable =
-            candidateResults.flatMap( candidates -> Observable.from( candidates ) );
-
-        //since we'll only have at most 100 results in memory at a time, we roll up our groups and emit them on to
-        // the collector
-        final Observable<CandidateGroup> candidateGroup =
-            candidateObservable.groupBy( candidate -> candidate.getId() ).map( observableGroup -> {
-
-
-                //for each group, create a list, then sort by highest version first
-                final List<CandidateResult> groupList = observableGroup.toList().toBlocking().last();
-
-                Collections.sort( groupList, CandidateVersionComparator::compare );
-
-                //create our candidate group and emit it
-                final CandidateGroup group =
-                    new CandidateGroup( groupList.get( 0 ), groupList.subList( 1, groupList.size() ) );
-
-                return group;
-            } );
-
+        //load all the potential candidates
+        final Observable<CandidateCollector> collectedCandidates = idObservable.compose( createCandidates() );
 
-        //buffer our candidate group up to our resultset size.
-        final Observable<CandidateCollector> collectedCandidates =
-            candidateGroup.buffer( resultSetSize ).flatMap( candidates -> {
+        //now we have our collected candidates, load them from ES
+        final Observable<SearchResults> loadedEntities = collectedCandidates.map( loadEntities( resultSetSize ) );
 
-                final Observable<CandidateCollector> collector = Observable.from( candidates ).collect(
-                    () -> new CandidateCollector( resultSetSize ), ( candidateCollector, candidate ) -> {
-                        //add our candidates to our collector
-                        candidateCollector.addCandidate( candidate.toKeep );
-                        //add our empty results
-                        candidateCollector.addEmptyResults( candidate.toRemove );
-                    } );
-
-                return collector;
-            } );
 
-        //now we have our collected candidates, load them
+        //after we've loaded, remove everything that's left
+        loadedEntities.doOnNext( results ->{
 
+            //run the deindex
+            Observable.from( results.toRemove ).collect( () -> index.createBatch(), (batch, toRemove) -> {
+              batch.deindex( toRemove.indexScope, toRemove.result );
+            }).doOnNext( batch -> batch.execute() ).toBlocking().lastOrDefault( null );
 
-        final Observable<SearchResults> loadedEntities = collectedCandidates.map( loadEntities(resultSetSize) );
+        });
 
 
         return loadedEntities;
     }
 
 
-    /**
-     * Perform the initial search with the sourceId
-     */
-    private Func0<CandidateResults> initialSearch( final Id sourceId ) {
-        return () -> index.search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ) );
-    }
 
 
     /**
-     * Search the next page for the specified source
+     * Using an input of observable ids, transform and collect them into a set of CandidateCollector to be loaded, or
+     * verified by the EM
      */
-    private Func1<String, CandidateResults> nextPage( final Id sourceId ) {
-        return cursor -> index
-            .search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ).withCursor( cursor ) );
-    }
+    private Observable.Transformer<Id, CandidateCollector> createCandidates() {
+        return idObservable -> {
 
+            //create our observable of candidate search results
+            final Observable<ScopedCandidateResults> candidateResults = idObservable.flatMap(
 
-    /**
-     * Function that will load our entities from our candidates, filter stale or missing candidates and return results
-     * @param expectedSize
-     * @return
-     */
-    private Func1<CandidateCollector, SearchResults> loadEntities(final int expectedSize) {
-        return candidateCollector -> {
 
-            //from our candidates, group them by id type so we can create scopes
-            Observable.from( candidateCollector.getCandidates() ).groupBy( candidate -> candidate.getId().getType() )
-                      .flatMap( groups -> {
+                id -> {
 
+                    //create the index scope from the id
+                    final IndexScope scope = createScope( id );
 
-                          final List<CandidateResult> candidates = groups.toList().toBlocking().last();
+                    //perform the ES search of candidate results
+                    final Observable<ScopedCandidateResults> scoped = Observable.create(
+                        new ElasticSearchObservable( initialSearch( scope ), nextPage( scope ) ) );
 
-                          //we can get no results, so quit aggregating if there are none
-                          if ( candidates.size() == 0 ) {
-                              return Observable.just( new SearchResults( 0 ) );
-                          }
 
+                    final Observable<CandidateCollector> collectedResults =  scoped.flatMap( scopedCandidateResults -> {
 
-                          final String typeName = candidates.get( 0 ).getId().getType();
+                        //here b/c the compiler cannot infer type directly
+                        final Observable<CandidateResult> candidates =
+                            Observable.from( scopedCandidateResults.candidateResults );
 
-                          final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType( typeName );
+                        return candidates.collect( () -> new CandidateCollector( resultSetSize,
+                            scopedCandidateResults.candidateResults.getCursor() ), ( collector, candidate ) -> {
+                            collector.insert( candidate );
+                        } );
+                    } );
 
 
-                          //create our scope to get our entity manager
 
-                          final CollectionScope scope =
-                              new CollectionScopeImpl( applicationId, applicationId, collectionType );
+                    //group our candidates by type
+//                   collectedResults.flatMap( collected -> Observable.from( collected.candidates ) ).groupBy( candidate -> candidate.getId().getType() ).co
 
-                          final EntityCollectionManager ecm =
-                              entityCollectionManagerFactory.createCollectionManager( scope );
 
+                    final Observable<SearchResults> resultsObservable = collectedResults.map( collectedCandidates -> {
 
-                          //get our entity ids
-                          final List<Id> entityIds =
-                              Observable.from( candidates ).map( c -> c.getId() ).toList().toBlocking().last();
 
-                          //TODO, change this out
+                        //we can get no results, so quit aggregating if there are none
+                        if ( collectedCandidates.candidates.size() == 0 ) {
+                            return new SearchResults( 0, collectedCandidates.cursor );
+                        }
 
-                          //an observable of our entity set
 
+                        final String collectionType =
+                            CpNamingUtils.getCollectionScopeNameFromEntityType( id.getType() );
 
 
-                          //now go through all our candidates and verify
+                        //create our scope to get our entity manager
 
-                          return Observable.from( candidates ).collect(  () -> new SearchResults( expectedSize ), (searchResults, candidate) ->{
+                        final CollectionScope collectionScope =
+                            new CollectionScopeImpl( applicationId, applicationId, collectionType );
 
-                              final EntitySet entitySet = ecm.load( entityIds ).toBlocking().last();
+                        final EntityCollectionManager ecm =
+                            entityCollectionManagerFactory.createCollectionManager( collectionScope );
 
-                              final MvccEntity entity = entitySet.getEntity( candidate.getId() );
 
+                        //get our entity ids
+                        final List<Id> entityIds =
+                            Observable.from( collectedCandidates.candidates ).map( c -> c.getId() ).toList()
+                                      .toBlocking().last();
 
-                              //our candidate it stale, or target entity was deleted add it to the remove of our collector
-                              if(UUIDComparator.staticCompare( entity.getVersion(), candidate.getVersion()) > 0 || !entity.getEntity().isPresent()){
-                                  searchResults.addToRemove( candidate );
-                                  return;
-                              }
+                        //TODO, change this out
 
+                        //an observable of our entity set
 
-                              searchResults.addEntity( entity.getEntity().get() );
+                        return ecm.load( entityIds ).map( results -> {
 
+                            Observable.from( entityIds ).collect( () -> new SearchResults( resultSetSize ),
+                                ( searchResults, entityIds ) -> {
+                                } );
+                        } );
+                    } );
 
-                          } )
-                              //add the existing set to remove to this set
-                              .doOnNext( results -> results.addToRemove( candidateCollector.getToRemove() ) );
+                    return resultsObservable;
+                } );
 
-                      } );
+            return candidateResults;
+        };
+    }
 
 
-            return null;
-        };
+    /**
+     * Perform the initial search with the sourceId
+     */
+    private Func0<ScopedCandidateResults> initialSearch( final IndexScope scope ) {
+        return () -> new ScopedCandidateResults(scope,  index.search(  scope, types, Query.fromQL( query ) ) );
+    }
+
+
+    /**
+     * Search the next page for the specified source
+     */
+    private Func1<String, ScopedCandidateResults> nextPage( final IndexScope scope  ) {
+        return cursor -> new ScopedCandidateResults(scope,   index.search( scope, types, Query.fromQL( query ).withCursor(
+            cursor ) ) );
     }
 
 
+    /**
+     * Create the scope and return it
+     * @param sourceId
+     * @return
+     */
+    private IndexScope createScope(final Id sourceId){
+        return new IndexScopeImpl( sourceId, edgeName );
+    }
+
+//
+//    /**
+//     * Function that will load our entities from our candidates, filter stale or missing candidates and return results
+//     */
+//    private Func1<CandidateCollector, SearchResults> loadEntities( final int expectedSize ) {
+//        return candidateCollector -> {
+//
+//            //from our candidates, group them by id type so we can create scopes
+//            Observable.from( candidateCollector.getCandidates() ).groupBy( candidate -> candidate.result.getId().getType() )
+//                      .flatMap( groups -> {
+//
+//
+//                          final List<ScopedCandidateResult> candidates = groups.toList().toBlocking().last();
+//
+//                          //we can get no results, so quit aggregating if there are none
+//                          if ( candidates.size() == 0 ) {
+//                              return Observable.just( new SearchResults(  0, null ) );
+//                          }
+//
+//
+//                          final String typeName = candidates.get( 0 ).result.getId().getType();
+//
+//                          final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType( typeName );
+//
+//
+//                          //create our scope to get our entity manager
+//
+//                          final CollectionScope scope =
+//                              new CollectionScopeImpl( applicationId, applicationId, collectionType );
+//
+//                          final EntityCollectionManager ecm =
+//                              entityCollectionManagerFactory.createCollectionManager( scope );
+//
+//
+//                          //get our entity ids
+//                          final List<Id> entityIds =
+//                              Observable.from( candidates ).map( c -> c.result.getId() ).toList().toBlocking().last();
+//
+//                          //TODO, change this out
+//
+//                          //an observable of our entity set
+//
+//
+//                          //now go through all our candidates and verify
+//
+//                          return Observable.from( candidates ).collect( () -> new SearchResults(
+//                                  expectedSize ),
+//                              ( searchResults, candidate ) -> {
+//
+//                                  final EntitySet entitySet = ecm.load( entityIds ).toBlocking().last();
+//
+//                                  final MvccEntity entity = entitySet.getEntity( candidate.result.getId() );
+//
+//
+//                                  //our candidate it stale, or target entity was deleted add it to the remove of our
+//                                  // collector
+//                                  if ( UUIDComparator.staticCompare( entity.getVersion(), candidate.result.getVersion() ) > 0
+//                                      || !entity.getEntity().isPresent() ) {
+//
+//                                      searchResults.addToRemove( candidate  );
+//                                      return;
+//                                  }
+//
+//
+//                                  searchResults.addEntity( entity.getEntity().get() );
+//                              } )
+//                              //add the existing set to remove to this set
+//                              .doOnNext( results -> results.addToRemove( candidateCollector.getToRemove() ) );
+//                      } );
+//
+//
+//            return null;
+//        };
+//    }
 
 
     /**
@@ -238,20 +307,49 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
         private final List<CandidateResult> candidates;
         private final List<CandidateResult> toRemove;
 
+        private final Map<Id, Integer> indexMapping;
 
-        public CandidateCollector( final int maxSize ) {
+        private final String cursor;
+
+
+        public CandidateCollector( final int maxSize, final String cursor ) {
+            this.cursor = cursor;
             candidates = new ArrayList<>( maxSize );
             toRemove = new ArrayList<>( maxSize );
+            indexMapping = new HashMap<>(maxSize);
         }
 
 
-        public void addCandidate( final CandidateResult candidate ) {
-            this.candidates.add( candidate );
-        }
 
+        public void insert(final CandidateResult newValue){
+
+            final Id candidateId = newValue.getId();
+
+            final Integer index = indexMapping.get( candidateId  );
+
+            if(index == null){
+                candidates.add( newValue );
+                indexMapping.put( candidateId, candidates.size()-1);
+                return;
+            }
+
+            //present, perform a comparison
+
+            final CandidateResult existing = candidates.get( index );
+
+
+            //it's a greater version, add this to ignore
+            if(UUIDComparator.staticCompare( existing.getVersion(), newValue.getVersion() ) > 0){
+                toRemove.add( newValue );
+            }
+
+            //remove the stale version from the list and put it in deindex
+            else{
+                candidates.remove( index );
+                candidates.add( newValue );
+                toRemove.add( existing );
+            }
 
-        public void addEmptyResults( final Collection<CandidateResult> stale ) {
-            this.toRemove.addAll( stale );
         }
 
 
@@ -268,13 +366,16 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
 
     public static class SearchResults {
         private final List<Entity> entities;
-        private final List<CandidateResult> toRemove;
+        private final List<ScopedCandidateResult> toRemove;
+        private final int maxSize;
 
-        private String cursor;
+        private final String cursor;
 
 
-        public SearchResults( final int maxSize ) {
-            entities = new ArrayList<>( maxSize );
+        public SearchResults( final int maxSize, final String cursor ) {
+            this.maxSize = maxSize;
+            this.cursor = cursor;
+            this.entities = new ArrayList<>( maxSize );
             this.toRemove = new ArrayList<>( maxSize );
         }
 
@@ -284,20 +385,23 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
         }
 
 
-        public void addToRemove( final Collection<CandidateResult> stale ) {
+        public void addToRemove( final Collection<ScopedCandidateResult> stale ) {
             this.toRemove.addAll( stale );
         }
 
 
-        public void addToRemove( final CandidateResult candidateResult ) {
-                   this.toRemove.add( candidateResult );
-               }
+        public void addToRemove( final ScopedCandidateResult candidateResult ) {
+            this.toRemove.add( candidateResult );
+        }
 
 
+        public String getCursor() {
+            return cursor;
+        }
 
 
-        public void setCursor( final String cursor ) {
-            this.cursor = cursor;
+        public boolean isFull(){
+            return entities.size() >= maxSize;
         }
     }
 
@@ -305,26 +409,26 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
     /**
      * An observable that will perform a search and continually emit results while they exist.
      */
-    public static class ElasticSearchObservable implements Observable.OnSubscribe<CandidateResults> {
+    public static class ElasticSearchObservable implements Observable.OnSubscribe<ScopedCandidateResults> {
 
-        private final Func1<String, CandidateResults> fetchNextPage;
-        private final Func0<CandidateResults> fetchInitialResults;
+        private final Func1<String, ScopedCandidateResults> fetchNextPage;
+        private final Func0<ScopedCandidateResults> fetchInitialResults;
 
 
-        public ElasticSearchObservable( final Func0<CandidateResults> fetchInitialResults,
-                                        final Func1<String, CandidateResults> fetchNextPage ) {
+        public ElasticSearchObservable( final Func0<ScopedCandidateResults> fetchInitialResults,
+                                        final Func1<String, ScopedCandidateResults> fetchNextPage ) {
             this.fetchInitialResults = fetchInitialResults;
             this.fetchNextPage = fetchNextPage;
         }
 
 
         @Override
-        public void call( final Subscriber<? super CandidateResults> subscriber ) {
+        public void call( final Subscriber<? super ScopedCandidateResults> subscriber ) {
 
             subscriber.onStart();
 
             try {
-                CandidateResults results = fetchInitialResults.call();
+                ScopedCandidateResults results = fetchInitialResults.call();
 
 
                 //emit our next page
@@ -332,13 +436,13 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
                     subscriber.onNext( results );
 
                     //if we have no cursor, we're done
-                    if ( !results.hasCursor() ) {
+                    if ( !results.candidateResults.hasCursor() ) {
                         break;
                     }
 
 
                     //we have a cursor, get our results to emit for the next page
-                    results = fetchNextPage.call( results.getCursor() );
+                    results = fetchNextPage.call( results.candidateResults.getCursor() );
                 }
 
                 subscriber.onCompleted();
@@ -351,16 +455,51 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
 
 
     /**
-     * A message that contains the candidate to keep, and the candidate toRemove
+     * A candidate result, aslong with the scope it was returned in
+     */
+    public static class ScopedCandidateResult{
+        private final IndexScope indexScope;
+        private final CandidateResult result;
+
+
+        public ScopedCandidateResult( final IndexScope indexScope, final CandidateResult result ) {
+            this.indexScope = indexScope;
+            this.result = result;
+        }
+    }
+
+
+    /**
+     * Object that represents our candidate results, along with the scope it was searched in
+     */
+    public static class ScopedCandidateResults {
+        private final IndexScope indexScope;
+        private CandidateResults candidateResults;
+
+
+        public ScopedCandidateResults( final IndexScope indexScope, final CandidateResults candidateResults ) {
+            this.indexScope = indexScope;
+            this.candidateResults = candidateResults;
+        }
+    }
+
+
+    /**
+     * A message that contains the candidate to keep, and the candidate toRemove and the scope they were searched in
      */
     public static class CandidateGroup {
+        private final IndexScope indexScope;
         private final CandidateResult toKeep;
         private final Collection<CandidateResult> toRemove;
+        private final String cursor;
 
 
-        public CandidateGroup( final CandidateResult toKeep, final Collection<CandidateResult> toRemove ) {
+        public CandidateGroup( final IndexScope indexScope, final CandidateResult toKeep,
+                               final Collection<CandidateResult> toRemove, final String cursor ) {
+            this.indexScope = indexScope;
             this.toKeep = toKeep;
             this.toRemove = toRemove;
+            this.cursor = cursor;
         }
     }
 
@@ -375,10 +514,5 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
         }
     }
 
-    /***********************
-     * FROM HERE DOWN IS EXPERIMENTAL
-     *************************
-     */
-
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cee76da7/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
index 8a0c014..93de094 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
@@ -38,7 +38,7 @@ public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
     final Func1<R, Boolean> shortCircuitWhen;
 
 
-    public CollectUntil( final Func1<R, Boolean> shortCircuitWhen,  final Func0<R> stateFactory, final Action2<R, ? super T> collector) {
+    public CollectUntil(  final Func0<R> stateFactory,final Func1<R, Boolean> shortCircuitWhen,  final Action2<R, ? super T> collector) {
         this.stateFactory = stateFactory;
         this.collector = collector;
         this.shortCircuitWhen = shortCircuitWhen;


[4/5] incubator-usergrid git commit: Refactor to clean up. Still a WIP

Posted by to...@apache.org.
Refactor to clean up.  Still a WIP


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/61aa9794
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/61aa9794
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/61aa9794

Branch: refs/heads/USERGRID-528
Commit: 61aa97945313533f6ef676f3f9dab5a22b9cb63b
Parents: c4f6548
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 23 09:52:44 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 23 17:23:46 2015 -0600

----------------------------------------------------------------------
 .../corepersistence/io/read/Command.java        |   8 +-
 .../corepersistence/io/read/CommandBuilder.java |   8 +-
 .../io/read/EntityIndexCommand.java             | 384 +++++++++++++++++++
 .../io/read/EntityIndexCommands.java            | 119 ------
 .../corepersistence/rx/impl/CollectUntil.java   |  11 +-
 .../java/org/apache/usergrid/TempExample.java   | 217 ++++++++++-
 .../rx/impl/CollectUntilTest.java               |   6 +-
 7 files changed, 605 insertions(+), 148 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
index 4e3b0fc..34b8d1e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
@@ -25,14 +25,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
 import rx.Observable;
 
 
-public interface Command<T> {
+public interface Command<T, R> extends Observable.Transformer<T, R> {
 
 
-    /**
-     * Process our input stream
-     * @param input
-     * @return
-     */
-    public Observable<T> process( Observable<T> input );
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
index 828cbc9..bbf74b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
@@ -38,7 +38,7 @@ public class CommandBuilder {
 
 
     private CursorCache cache;
-    private final Observable<Id> pathObservable;
+    private Observable<Id> pathObservable;
 
 
     public CommandBuilder( final Id root ) {
@@ -46,6 +46,12 @@ public class CommandBuilder {
     }
 
 
+
+    public void addCommand(final Command<Id, Id> intermediateCommand){
+       pathObservable =  pathObservable.compose( intermediateCommand );
+    }
+
+
     /**
      * Set our cache
      * @param cache

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
new file mode 100644
index 0000000..a8fa221
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+import rx.Subscriber;
+import rx.functions.Func0;
+import rx.functions.Func1;
+
+
+@Singleton
+public class EntityIndexCommand implements Command<Id, EntityIndexCommand.SearchResults> {
+
+    private final Id applicationId;
+    private final ApplicationScope applicationScope;
+    private final ApplicationEntityIndex index;
+    private final SearchTypes types;
+    private final String query;
+    private final int resultSetSize;
+    private final String scopeType;
+    private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+    @Inject
+    public EntityIndexCommand( final Id applicationId, final ApplicationScope applicationScope,
+                               final ApplicationEntityIndex index, final SearchTypes types, final String query,
+                               final int resultSetSize, final String scopeType,
+                               final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+        this.applicationId = applicationId;
+        this.applicationScope = applicationScope;
+
+        this.index = index;
+        this.types = types;
+        this.query = query;
+        this.resultSetSize = resultSetSize;
+        this.scopeType = scopeType;
+        this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+    }
+
+
+    @Override
+    public Observable<EntityIndexCommand.SearchResults> call( final Observable<Id> idObservable ) {
+
+        //create our observable of candidate search results
+        final Observable<CandidateResults> candidateResults = idObservable
+            .flatMap( id -> Observable.create( new ElasticSearchObservable( initialSearch( id ), nextPage( id ) ) ) );
+
+        final Observable<CandidateResult> candidateObservable =
+            candidateResults.flatMap( candidates -> Observable.from( candidates ) );
+
+        //since we'll only have at most 100 results in memory at a time, we roll up our groups and emit them on to
+        // the collector
+        final Observable<CandidateGroup> candidateGroup =
+            candidateObservable.groupBy( candidate -> candidate.getId() ).map( observableGroup -> {
+
+
+                //for each group, create a list, then sort by highest version first
+                final List<CandidateResult> groupList = observableGroup.toList().toBlocking().last();
+
+                Collections.sort( groupList, CandidateVersionComparator::compare );
+
+                //create our candidate group and emit it
+                final CandidateGroup group =
+                    new CandidateGroup( groupList.get( 0 ), groupList.subList( 1, groupList.size() ) );
+
+                return group;
+            } );
+
+
+        //buffer our candidate group up to our resultset size.
+        final Observable<CandidateCollector> collectedCandidates =
+            candidateGroup.buffer( resultSetSize ).flatMap( candidates -> {
+
+                final Observable<CandidateCollector> collector = Observable.from( candidates ).collect(
+                    () -> new CandidateCollector( resultSetSize ), ( candidateCollector, candidate ) -> {
+                        //add our candidates to our collector
+                        candidateCollector.addCandidate( candidate.toKeep );
+                        //add our empty results
+                        candidateCollector.addEmptyResults( candidate.toRemove );
+                    } );
+
+                return collector;
+            } );
+
+        //now we have our collected candidates, load them
+
+
+        final Observable<SearchResults> loadedEntities = collectedCandidates.map( loadEntities(resultSetSize) );
+
+
+        return loadedEntities;
+    }
+
+
+    /**
+     * Perform the initial search with the sourceId
+     */
+    private Func0<CandidateResults> initialSearch( final Id sourceId ) {
+        return () -> index.search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ) );
+    }
+
+
+    /**
+     * Search the next page for the specified source
+     */
+    private Func1<String, CandidateResults> nextPage( final Id sourceId ) {
+        return cursor -> index
+            .search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ).withCursor( cursor ) );
+    }
+
+
+    /**
+     * Function that will load our entities from our candidates, filter stale or missing candidates and return results
+     * @param expectedSize
+     * @return
+     */
+    private Func1<CandidateCollector, SearchResults> loadEntities(final int expectedSize) {
+        return candidateCollector -> {
+
+            //from our candidates, group them by id type so we can create scopes
+            Observable.from( candidateCollector.getCandidates() ).groupBy( candidate -> candidate.getId().getType() )
+                      .flatMap( groups -> {
+
+
+                          final List<CandidateResult> candidates = groups.toList().toBlocking().last();
+
+                          //we can get no results, so quit aggregating if there are none
+                          if ( candidates.size() == 0 ) {
+                              return Observable.just( new SearchResults( 0 ) );
+                          }
+
+
+                          final String typeName = candidates.get( 0 ).getId().getType();
+
+                          final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType( typeName );
+
+
+                          //create our scope to get our entity manager
+
+                          final CollectionScope scope =
+                              new CollectionScopeImpl( applicationId, applicationId, collectionType );
+
+                          final EntityCollectionManager ecm =
+                              entityCollectionManagerFactory.createCollectionManager( scope );
+
+
+                          //get our entity ids
+                          final List<Id> entityIds =
+                              Observable.from( candidates ).map( c -> c.getId() ).toList().toBlocking().last();
+
+                          //TODO, change this out
+
+                          //an observable of our entity set
+
+
+
+                          //now go through all our candidates and verify
+
+                          return Observable.from( candidates ).collect(  () -> new SearchResults( expectedSize ), (searchResults, candidate) ->{
+
+                              final EntitySet entitySet = ecm.load( entityIds ).toBlocking().last();
+
+                              final MvccEntity entity = entitySet.getEntity( candidate.getId() );
+
+
+                              //our candidate it stale, or target entity was deleted add it to the remove of our collector
+                              if(UUIDComparator.staticCompare( entity.getVersion(), candidate.getVersion()) > 0 || !entity.getEntity().isPresent()){
+                                  searchResults.addToRemove( candidate );
+                                  return;
+                              }
+
+
+                              searchResults.addEntity( entity.getEntity().get() );
+
+
+                          } )
+                              //add the existing set to remove to this set
+                              .doOnNext( results -> results.addToRemove( candidateCollector.getToRemove() ) );
+
+                      } );
+
+
+            return null;
+        };
+    }
+
+
+
+
+    /**
+     * Collects all valid results (in order) as well as candidates to be removed
+     */
+    public static class CandidateCollector {
+        private final List<CandidateResult> candidates;
+        private final List<CandidateResult> toRemove;
+
+
+        public CandidateCollector( final int maxSize ) {
+            candidates = new ArrayList<>( maxSize );
+            toRemove = new ArrayList<>( maxSize );
+        }
+
+
+        public void addCandidate( final CandidateResult candidate ) {
+            this.candidates.add( candidate );
+        }
+
+
+        public void addEmptyResults( final Collection<CandidateResult> stale ) {
+            this.toRemove.addAll( stale );
+        }
+
+
+        public List<CandidateResult> getCandidates() {
+            return candidates;
+        }
+
+
+        public List<CandidateResult> getToRemove() {
+            return toRemove;
+        }
+    }
+
+
+    public static class SearchResults {
+        private final List<Entity> entities;
+        private final List<CandidateResult> toRemove;
+
+        private String cursor;
+
+
+        public SearchResults( final int maxSize ) {
+            entities = new ArrayList<>( maxSize );
+            this.toRemove = new ArrayList<>( maxSize );
+        }
+
+
+        public void addEntity( final Entity entity ) {
+            this.entities.add( entity );
+        }
+
+
+        public void addToRemove( final Collection<CandidateResult> stale ) {
+            this.toRemove.addAll( stale );
+        }
+
+
+        public void addToRemove( final CandidateResult candidateResult ) {
+                   this.toRemove.add( candidateResult );
+               }
+
+
+
+
+        public void setCursor( final String cursor ) {
+            this.cursor = cursor;
+        }
+    }
+
+
+    /**
+     * An observable that will perform a search and continually emit results while they exist.
+     */
+    public static class ElasticSearchObservable implements Observable.OnSubscribe<CandidateResults> {
+
+        private final Func1<String, CandidateResults> fetchNextPage;
+        private final Func0<CandidateResults> fetchInitialResults;
+
+
+        public ElasticSearchObservable( final Func0<CandidateResults> fetchInitialResults,
+                                        final Func1<String, CandidateResults> fetchNextPage ) {
+            this.fetchInitialResults = fetchInitialResults;
+            this.fetchNextPage = fetchNextPage;
+        }
+
+
+        @Override
+        public void call( final Subscriber<? super CandidateResults> subscriber ) {
+
+            subscriber.onStart();
+
+            try {
+                CandidateResults results = fetchInitialResults.call();
+
+
+                //emit our next page
+                while ( true ) {
+                    subscriber.onNext( results );
+
+                    //if we have no cursor, we're done
+                    if ( !results.hasCursor() ) {
+                        break;
+                    }
+
+
+                    //we have a cursor, get our results to emit for the next page
+                    results = fetchNextPage.call( results.getCursor() );
+                }
+
+                subscriber.onCompleted();
+            }
+            catch ( Throwable t ) {
+                subscriber.onError( t );
+            }
+        }
+    }
+
+
+    /**
+     * A message that contains the candidate to keep, and the candidate toRemove
+     */
+    public static class CandidateGroup {
+        private final CandidateResult toKeep;
+        private final Collection<CandidateResult> toRemove;
+
+
+        public CandidateGroup( final CandidateResult toKeep, final Collection<CandidateResult> toRemove ) {
+            this.toKeep = toKeep;
+            this.toRemove = toRemove;
+        }
+    }
+
+
+    /**
+     * Compares 2 candidates by version.  The max version is considered greater
+     */
+    private static final class CandidateVersionComparator {
+
+        public static int compare( final CandidateResult o1, final CandidateResult o2 ) {
+            return UUIDComparator.staticCompare( o1.getVersion(), o2.getVersion() );
+        }
+    }
+
+    /***********************
+     * FROM HERE DOWN IS EXPERIMENTAL
+     *************************
+     */
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
deleted file mode 100644
index d6f4e93..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.io.read;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
-import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-import rx.functions.Action2;
-import rx.functions.Func1;
-
-
-public class EntityIndexCommands {
-
-
-//    /**
-//     * Perform a search of all the entities, and then return the observable of search results
-//     * @param index
-//     * @param edgeType
-//     * @param types
-//     * @param query
-//     * @return
-//     */
-//    public static Func1<Id, SearchResults> searchEntities(final ApplicationEntityIndex index,final String edgeType, final SearchTypes types, final String query  ){
-//
-//        return nodeId -> {
-//
-//        }
-//    }
-    /**
-     * Construct an indexScope from the input id type
-     * @param type
-     * @return
-     */
-   public static Func1<Id, IndexScope> createSearchScope(final String type){
-       return id -> new IndexScopeImpl( id, type );
-   }
-    /**
-     * Get our candidate results
-     * @param index
-     * @param types The types to return
-     * @param query
-     * @return
-     */
-    public static Func1<IndexScope, CandidateResults> getCandidates(final ApplicationEntityIndex index, final SearchTypes types, final String query){
-        return indexScope -> index.search( indexScope, types, Query.fromQLNullSafe( query ) );
-    }
-
-
-    /**
-     * Flattens candidate results into a single stream of a result
-     * @return
-     */
-    public static Func1<CandidateResults, Observable<CandidateResult>> flattenCandidates(){
-        return (CandidateResults candidateResults) -> Observable.from( candidateResults );
-    }
-
-
-    public static Action2<SearchResults, EntitySet> collectSet(){
-        return (searchResults, entitySet) -> {
-          searchResults.addEntities( entitySet.entities );
-        };
-    }
-
-
-
-    public static class SearchResults{
-        private final List<Entity> entities;
-        private String cursor;
-
-
-        public SearchResults(final int maxSize) {entities = new ArrayList<>(maxSize);}
-
-        public void addEntities(final Collection<Entity> entities){
-            this.entities.addAll( entities );
-
-        }
-        public void setCursor(final String cursor){
-            this.cursor = cursor;
-        }
-    }
-
-    public static class EntitySet{
-        private final List<Entity> entities;
-
-
-        public EntitySet( final List<Entity> entities ) {this.entities = entities;}
-    }
-
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
index a2cb754..8a0c014 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
@@ -29,20 +29,19 @@ import rx.internal.operators.OperatorScan;
 
 
 /**
- * An operation for performing a collect until the predicate returns true
+ * An operation for performing a collect until the shortCircuitWhen returns true
  */
 public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
 
     final Func0<R> stateFactory;
     final Action2<R, ? super T> collector;
-    final Func1<R, Boolean> predicate;
+    final Func1<R, Boolean> shortCircuitWhen;
 
 
-    public CollectUntil( final Func0<R> stateFactory, final Action2<R, ? super T> collector,
-                          final Func1<R, Boolean> predicate ) {
+    public CollectUntil( final Func1<R, Boolean> shortCircuitWhen,  final Func0<R> stateFactory, final Action2<R, ? super T> collector) {
         this.stateFactory = stateFactory;
         this.collector = collector;
-        this.predicate = predicate;
+        this.shortCircuitWhen = shortCircuitWhen;
     }
 
 
@@ -54,7 +53,7 @@ public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
         };
 
 
-        return tObservable.lift( new OperatorScan<>( stateFactory, accumulator ) ).takeUntil( predicate );
+        return tObservable.lift( new OperatorScan<>( stateFactory, accumulator ) ).takeUntil( shortCircuitWhen );
     }
 }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/test/java/org/apache/usergrid/TempExample.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/TempExample.java b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
index 1db450d..ed58111 100644
--- a/stack/core/src/test/java/org/apache/usergrid/TempExample.java
+++ b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
@@ -20,16 +20,37 @@
 package org.apache.usergrid;
 
 
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.usergrid.corepersistence.rx.impl.CollectUntil;
 import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
 import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.model.entity.Id;
 
+import com.fasterxml.uuid.UUIDComparator;
+
 import rx.Observable;
+import rx.Statement;
+import rx.Subscriber;
+import rx.functions.Action2;
+import rx.functions.Func0;
+import rx.functions.Func1;
+import rx.observables.GroupedObservable;
 
 import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.createSearchScope;
 import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.getCandidates;
 import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
 
 
 public class TempExample {
@@ -37,38 +58,210 @@ public class TempExample {
     //set our root observable
 
 
-    public static void main(String[] args) {
 
-        final Id rootId = createId( "thing" );
+//
+//        final Id rootId = createId( "thing" );
+//
+//        final ApplicationEntityIndex index = null;
+//
+//
+//        final SearchTypes searchType = SearchTypes.fromTypes( "test" );
+//
+//        final String query = "select * ";
+//
+//        final int selectSize = 100;
+//
+//        final Observable<CandidateResults> observable = Observable.just( rootId ).map( createSearchScope( "type" ) )
+//                                                                  .map( getCandidates( index, searchType,
+//                                                                      query ) ); //.compose();
+//
+//
+//        final Observable<GroupedObservable<Id, CandidateResult>> next =
+//            observable.flatMap( candidates -> Observable.from( candidates ) ).groupBy( result -> result.getId() );
+//
+//
+//        final Observable<ResultsCollector> results = next.compose(  new CollectUntil<>(
+//            //stop when the collector has a full set
+//                    collector -> collector.hasFullSet(),
+//            //create a new results collector, based on requested size
+//            () -> new ResultsCollector( selectSize ),
+//            //collect each candidate into our set of Candidate Results
+//            ( collector, group ) -> {
+//
+//                    final List<CandidateResult> list = group.toList().toBlocking().last();
+//
+//                    collector.addCandidates( list );
+//                } ) );
+//
+
+        //now reduce these results again via a cassandra collector
+
+
+
+    @Test
+    public void testLoops(){
+
+       final Observable<Integer> observable =  Observable.create( new Observable.OnSubscribe<Integer>() {
+
+            int value = 0;
+
+
+            @Override
+            public void call( final Subscriber<? super Integer> subscriber ) {
+                subscriber.onNext( ++value );
+            }
+        } );
+
+
+
+
+        final CounterCollector collector = new CounterCollector();
+
+        final Action2<CounterCollector, Integer> collectorFunction  = (col, value) -> {col.set( value ); };
+
+
 
-        final ApplicationEntityIndex index = null;
+        final Func0<Boolean> complete =  () -> collector.isFull();
 
 
-        final SearchTypes searchType = SearchTypes.fromTypes( "test" );
+        final Observable<CounterCollector> collectorObservable = Statement.doWhile( observable,  complete ).collect( () -> collector, collectorFunction  );
 
-        final String query = "select * ";
 
-        final Observable<CandidateResults> observable = Observable.just( rootId ).map( createSearchScope( "type" ) ).map(getCandidates(index, searchType, query));
+        final int value = collectorObservable.toBlocking().last().lastValue;
 
 
-        observable.doOnNext( a -> System.out.println( a) ).toBlocking().last();
+        assertEquals(100, value);
+
+
+
+
+
+    }
+
+
+    private static class CounterCollector{
+        private int lastValue;
+
+        public void set(final int newValue){
+            lastValue = newValue;
+        }
+
+        public boolean isFull(){
+            return lastValue >= 100;
+        }
     }
 
 
-    private static final class ResultsCollector{
+    private static final class ResultsCollector {
+
+        private Map<Id, IndexResult<CandidateResult>> toKeep = new HashMap<>();
+        private Set<CandidateResult> toRemove = new HashSet<>();
+
+        private final int resultSetSize;
+
+
+        private int currentIndex;
+
+
+        private ResultsCollector( final int resultSetSize ) {this.resultSetSize = resultSetSize;}
+
 
         /**
          * Add the candidates to our collection
-         * @param results
          */
-        public void addCandidates(final CandidateResults results ){
+        public void addCandidates( final List<CandidateResult> results ) {
+
+            Collections.sort( results, CandidateVersionComparator::compare );
+
+            final CandidateResult maxCandidate = results.get( 0 );
+
+            //add everything we don't use to our toRemoveSet
+            for ( int i = 1; i < results.size(); i++ ) {
+                toRemove.add( results.get( i ) );
+            }
+
+            //we have this id already, remove it and pick the max
+            final Id maxId = maxCandidate.getId();
+
+            //see if it exists in our set to keep
+            final IndexResult<CandidateResult> existingCandidate = toKeep.get( maxId );
+
+            if ( existingCandidate != null ) {
+
+                final CandidateResult existing = existingCandidate.value;
+
+                //our new value is greater than our existing, replace it
+                if ( CandidateVersionComparator.compare( maxCandidate, existing ) > 0 ) {
+                    //add it to the keep
+                    toKeep.put( maxId, new IndexResult( currentIndex, maxCandidate ) );
 
-            //TODO, collect the results, removing groups
-            Observable.from( results ).groupBy( candidate -> candidate.getId() ).collect(  )
+                    //remove the old value
+                    toRemove.add( existingCandidate.value );
+                }
+
+                //what's in the map is already the max, add our candidate to the cleanup list
+                else {
+                    toRemove.add( maxCandidate );
+                }
+            }
+
+            //add it to our list of items to keep
+            else {
+                toKeep.put( maxId, new IndexResult( currentIndex, maxCandidate ) );
+            }
+
+
+            //increment our index for the next invocation
+            currentIndex++;
+        }
+
+
+        /**
+         * We have a full set to evaluate for candidates
+         */
+        public boolean hasFullSet() {
+            return resultSetSize >= toKeep.size();
         }
 
+
+        /**
+         * Get all the canidates we've collected
+         */
+        public Collection<IndexResult<CandidateResult>> getCandidates() {
+            return toKeep.values();
+        }
+
+        public Collection<CandidateResult> getStaleCandidates(){ return toRemove;}
     }
 
 
 
+
+    /**
+     * Compares 2 candidates by version.  The max version is considered greater
+     */
+    private static final class CandidateVersionComparator {
+
+        public static int compare( final CandidateResult o1, final CandidateResult o2 ) {
+            return UUIDComparator.staticCompare( o1.getVersion(), o2.getVersion() );
+        }
+    }
+
+
+    /**
+     * A data message with an index and a result.  Can be used in aggregations
+     * @param <T>
+     */
+    private static final class IndexResult<T>{
+        private final int index;
+
+        private final T value;
+
+
+        private IndexResult( final int index, final T value ) {
+            this.index = index;
+            this.value = value;
+        }
+    }
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
index ce12429..b6ebcaa 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
@@ -33,10 +33,10 @@ public class CollectUntilTest {
     public void testCollectUntil() {
 
         final CollectUntil<Integer, CountCollector> collectUntil =
-            new CollectUntil<>(
+            new CollectUntil<>(  collector -> collector.isFull() ,
                 () -> new CountCollector(),
-                ( collector, value ) -> collector.mark(),
-                collector -> collector.isFull() );
+                ( collector, value ) -> collector.mark()
+                );
 
 
         final CountCollector collector = Observable.range( 0, 200 ).compose( collectUntil ).toBlocking().last();