You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/01 17:27:44 UTC
[1/5] incubator-usergrid git commit: Initial commit. WIP overwrite
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-528 [created] cee76da72
Initial commit. WIP overwrite
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fe1fa85c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fe1fa85c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fe1fa85c
Branch: refs/heads/USERGRID-528
Commit: fe1fa85c36b5eb772c5ff03be816d71e1522793d
Parents: 61c5f9c
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Mar 19 09:47:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Mar 19 09:47:43 2015 -0600
----------------------------------------------------------------------
.../io/execution/CommandExecution.java | 37 +++++++++++
.../corepersistence/io/read/Command.java | 38 ++++++++++++
.../corepersistence/io/read/CommandBuilder.java | 65 ++++++++++++++++++++
.../io/reduce/StreamReducer.java | 38 ++++++++++++
4 files changed, 178 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java
new file mode 100644
index 0000000..0aaa764
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/execution/CommandExecution.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.execution;
+
+
+import org.apache.usergrid.corepersistence.io.read.CommandBuilder;
+import org.apache.usergrid.corepersistence.io.reduce.StreamReducer;
+
+
+public interface CommandExecution {
+
+ /**
+ * Invoke the commands, then feed them to the reducer to return a final output
+ * @param builder
+ * @param reducer
+ * @param <T>
+ * @return
+ */
+ public <T> T invoke(CommandBuilder builder, StreamReducer<T> reducer);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
new file mode 100644
index 0000000..4e3b0fc
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+public interface Command<T> {
+
+
+ /**
+ * Process our input stream
+ * @param input
+ * @return
+ */
+ public Observable<T> process( Observable<T> input );
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
new file mode 100644
index 0000000..119fc0e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.usergrid.corepersistence.io.reduce.StreamReducer;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+import rx.functions.Func1;
+
+
+/**
+ * The builder to hold the list of traversal commands
+ */
+public class CommandBuilder {
+
+ private final Id root;
+ private final List<Command<Id>> commandList;
+
+
+ public CommandBuilder( final Id root ) {this.root = root;
+ commandList = new ArrayList<>( );
+ }
+
+ public void addIntermediateCommand(final Command<Id> command){
+ commandList.add( command );
+ }
+
+
+ public <T> void addFinalCommand( final Command<T> command, final StreamReducer<T> reducer ) {
+
+ Observable.just("foo").flatMap( new Func1<String, Observable<?>>() {
+ @Override
+ public Observable<?> call( final String s ) {
+ return null;
+ }
+ };
+ }
+
+
+ public List<Command<Id>> getCommands(){
+ return commandList;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fe1fa85c/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java
new file mode 100644
index 0000000..ae84bd8
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/reduce/StreamReducer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.reduce;
+
+
+import org.apache.usergrid.corepersistence.io.read.Command;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+
+public interface StreamReducer<T> {
+
+ /**
+ * Reduce the result of our stream into a list of nodes
+ *
+ * @param candidates The candidate ids for loading
+ * @return
+ */
+ public T reduce( Observable<Id> candidates );
+}
[2/5] incubator-usergrid git commit: Merge branch 'USERGRID-405' into
USERGRID-494
Posted by to...@apache.org.
Merge branch 'USERGRID-405' into USERGRID-494
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c59f092
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c59f092
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c59f092
Branch: refs/heads/USERGRID-528
Commit: 4c59f09222c00183e994c49ad3f3ed96938bcd22
Parents: fe1fa85 a40fb8d
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 20 15:54:53 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 20 15:54:53 2015 -0600
----------------------------------------------------------------------
.../main/dist/init_instance/init_rest_server.sh | 6 +-
stack/core/pom.xml | 54 +-
.../batch/service/SchedulerServiceImpl.java | 17 +-
.../AllApplicationsObservable.java | 39 ++
.../usergrid/corepersistence/CoreModule.java | 88 ++-
.../corepersistence/CpEntityManager.java | 43 +-
.../corepersistence/CpEntityManagerFactory.java | 160 ++---
.../corepersistence/CpManagerCache.java | 57 +-
.../corepersistence/CpRelationManager.java | 43 +-
.../usergrid/corepersistence/CpSetup.java | 6 +-
.../usergrid/corepersistence/CpWalker.java | 81 +--
.../usergrid/corepersistence/ManagerCache.java | 13 +-
.../events/EntityDeletedHandler.java | 3 +-
.../events/EntityVersionDeletedHandler.java | 76 +--
.../migration/CoreDataVersions.java | 43 ++
.../migration/CoreMigration.java | 53 ++
.../migration/CoreMigrationPlugin.java | 64 ++
.../migration/EntityDataMigration.java | 147 ----
.../migration/EntityTypeMappingMigration.java | 69 +-
.../migration/GraphShardVersionMigration.java | 152 -----
.../migration/MigrationModuleVersionPlugin.java | 133 ++++
.../migration/MigrationSystemVersions.java | 43 ++
.../corepersistence/migration/Versions.java | 14 -
.../results/FilteringLoader.java | 3 +-
.../rx/AllEntitiesInSystemObservable.java | 101 ---
.../rx/ApplicationObservable.java | 128 ----
.../rx/EdgesFromSourceObservable.java | 63 --
.../rx/EdgesToTargetObservable.java | 63 --
.../corepersistence/rx/TargetIdObservable.java | 66 --
.../rx/impl/AbstractGraphVisitorImpl.java | 107 +++
.../rx/impl/AllApplicationsObservableImpl.java | 142 ++++
.../rx/impl/AllEntitiesInSystemImpl.java | 62 ++
.../rx/impl/AllNodesInGraphImpl.java | 55 ++
.../corepersistence/util/CpEntityMapUtils.java | 19 +-
.../corepersistence/util/CpNamingUtils.java | 2 +
.../usergrid/persistence/EntityManager.java | 19 +-
.../persistence/EntityManagerFactory.java | 22 -
.../persistence/ObservableIterator.java | 1 +
.../cassandra/EntityManagerFactoryImpl.java | 28 -
.../cassandra/EntityManagerImpl.java | 20 +-
.../org/apache/usergrid/CoreApplication.java | 24 +-
.../java/org/apache/usergrid/CoreITSetup.java | 6 +
.../org/apache/usergrid/CoreITSetupImpl.java | 27 +-
.../org/apache/usergrid/TestEntityIndex.java | 27 +
.../corepersistence/StaleIndexCleanupTest.java | 87 +--
.../migration/EntityDataMigrationIT.java | 262 -------
.../migration/EntityTypeMappingMigrationIT.java | 183 ++---
.../migration/GraphShardVersionMigrationIT.java | 226 ------
.../MigrationModuleVersionPluginTest.java | 259 +++++++
.../migration/MigrationTestRule.java | 99 ---
.../migration/TestProgressObserver.java | 71 --
.../rx/AllEntitiesInSystemObservableIT.java | 41 +-
.../rx/ApplicationObservableTestIT.java | 14 +-
.../rx/EdgesFromSourceObservableIT.java | 10 +-
.../rx/EdgesToTargetObservableIT.java | 8 +-
.../rx/TargetIdObservableTestIT.java | 7 +-
.../usergrid/persistence/CollectionIT.java | 80 +--
.../usergrid/persistence/CountingMutatorIT.java | 4 +-
.../persistence/EntityConnectionsIT.java | 14 +-
.../usergrid/persistence/EntityManagerIT.java | 16 +-
.../org/apache/usergrid/persistence/GeoIT.java | 27 +-
.../persistence/GeoQueryBooleanTest.java | 4 +-
.../apache/usergrid/persistence/IndexIT.java | 14 +-
.../usergrid/persistence/LargeEntityIT.java | 3 +-
.../usergrid/persistence/PathQueryIT.java | 6 +-
.../PerformanceEntityRebuildIndexTest.java | 39 +-
.../usergrid/persistence/PermissionsIT.java | 4 +-
.../cassandra/EntityManagerFactoryImplIT.java | 4 +-
.../persistence/query/CollectionIoHelper.java | 2 +-
.../persistence/query/ConnectionHelper.java | 2 +-
.../query/IntersectionTransitivePagingIT.java | 2 +-
.../query/IntersectionUnionPagingIT.java | 2 +-
.../persistence/query/IteratingQueryIT.java | 34 +-
.../resources/usergrid-custom-test.properties | 1 +
stack/corepersistence/collection/pom.xml | 14 +-
.../collection/EntityCollectionManager.java | 6 -
.../EntityCollectionManagerFactory.java | 27 +-
.../collection/EntityCollectionManagerSync.java | 49 --
.../persistence/collection/MvccEntity.java | 5 +-
.../persistence/collection/MvccLogEntry.java | 1 +
.../cache/CachedEntityCollectionManager.java | 7 -
.../collection/event/EntityVersionDeleted.java | 3 +-
.../collection/guice/CollectionModule.java | 45 +-
.../persistence/collection/guice/Write.java | 17 -
.../collection/guice/WriteUpdate.java | 17 -
.../EntityCollectionManagerFactoryImpl.java | 71 +-
.../impl/EntityCollectionManagerImpl.java | 129 +---
.../impl/EntityCollectionManagerSyncImpl.java | 71 --
.../collection/impl/EntityDeletedTask.java | 28 +-
.../impl/EntityVersionCleanupTask.java | 183 ++---
.../impl/EntityVersionCreatedTask.java | 26 +-
.../mvcc/MvccEntitySerializationStrategy.java | 102 ---
.../mvcc/stage/delete/MarkCommit.java | 2 +-
.../mvcc/stage/write/WriteCommit.java | 11 +-
.../collection/mvcc/stage/write/WriteStart.java | 6 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 8 +-
.../collection/serialization/EntityRepair.java | 38 --
.../MvccEntitySerializationStrategy.java | 119 ++++
.../serialization/OptimisticUpdate.java | 23 -
.../UniqueValueSerializationStrategy.java | 46 +-
.../serialization/UniqueValueSet.java | 4 +
.../impl/CollectionDataVersions.java | 44 ++
.../serialization/impl/EntityRepairImpl.java | 149 ----
.../impl/EntityVersionSerializer.java | 2 -
.../serialization/impl/FieldSerializer.java | 118 ----
.../MvccEntitySerializationStrategyImpl.java | 112 ++-
...vccEntitySerializationStrategyProxyImpl.java | 151 ++--
.../MvccEntitySerializationStrategyV1Impl.java | 9 +-
.../MvccEntitySerializationStrategyV2Impl.java | 8 +-
.../MvccEntitySerializationStrategyV3Impl.java | 587 ++++++++++++++++
.../serialization/impl/SerializationModule.java | 85 ++-
.../serialization/impl/UniqueFieldEntry.java | 61 ++
.../impl/UniqueFieldEntrySerializer.java | 141 ++++
.../impl/UniqueFieldRowKeySerializer.java | 122 ++++
.../UniqueValueSerializationStrategyImpl.java | 248 +++++--
.../impl/migration/CollectionMigration.java | 53 ++
.../migration/CollectionMigrationPlugin.java | 68 ++
.../impl/migration/EntityIdScope.java | 49 ++
.../migration/MvccEntityDataMigrationImpl.java | 260 +++++++
.../collection/util/EntityUtils.java | 71 --
.../collection/EntityCollectionManagerIT.java | 104 +--
.../EntityCollectionManagerSyncIT.java | 193 ------
.../collection/guice/TestCollectionModule.java | 22 +-
.../impl/EntityVersionCleanupTaskTest.java | 680 ++++++++-----------
.../impl/EntityVersionCreatedTaskTest.java | 2 -
.../mvcc/stage/AbstractEntityStageTest.java | 2 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 2 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 2 +-
.../mvcc/stage/write/FieldSerializerTest.java | 52 --
.../write/UniqueFieldRowKeySerializerTest.java | 52 ++
...niqueValueSerializationStrategyImplTest.java | 149 +++-
.../mvcc/stage/write/WriteCommitTest.java | 2 +-
.../mvcc/stage/write/WriteStartTest.java | 6 +-
.../persistence/collection/rx/ParallelTest.java | 10 +-
.../serialization/EntityRepairImplTest.java | 147 ----
.../impl/LogEntryIteratorTest.java | 3 +-
...MvccEntitySerializationStrategyImplTest.java | 121 +---
...cEntitySerializationStrategyProxyV1Test.java | 85 ---
...ntitySerializationStrategyProxyV1_3Test.java | 83 +++
...cEntitySerializationStrategyProxyV2Test.java | 83 ---
...ntitySerializationStrategyProxyV2_3Test.java | 84 +++
...ccEntitySerializationStrategyV1ImplTest.java | 80 ++-
...ccEntitySerializationStrategyV2ImplTest.java | 75 +-
.../MvccEntitySerializationStrategyV2Test.java | 7 +-
...ccEntitySerializationStrategyV3ImplTest.java | 74 ++
.../impl/SerializationComparison.java | 4 +-
.../impl/UniqueFieldEntrySerializerTest.java | 108 +++
...ctMvccEntityDataMigrationV1ToV3ImplTest.java | 215 ++++++
.../MvccEntityDataMigrationV1ToV3ImplTest.java | 105 +++
.../MvccEntityDataMigrationV2ToV3ImplTest.java | 99 +++
.../collection/util/InvalidEntityGenerator.java | 1 +
.../collection/util/LogEntryMock.java | 39 +-
.../collection/util/UniqueValueEntryMock.java | 161 +++++
.../collection/util/VersionGenerator.java | 55 ++
stack/corepersistence/common/pom.xml | 15 +-
.../usergrid/persistence/core/CPManager.java | 28 +
.../astyanax/DynamicCompositeParserImpl.java | 81 +++
.../core/astyanax/FieldBufferBuilder.java | 15 +
.../core/astyanax/FieldBufferParser.java | 13 +
.../astyanax/MultiKeyColumnNameIterator.java | 4 +-
.../core/astyanax/MultiRowColumnIterator.java | 12 +-
.../persistence/core/guice/CommonModule.java | 17 +-
.../persistence/core/guice/CurrentImpl.java | 42 --
.../persistence/core/guice/PreviousImpl.java | 42 --
.../core/hystrix/HystrixCassandra.java | 94 ---
.../migration/data/AbstractMigrationPlugin.java | 128 ++++
.../core/migration/data/DataMigration.java | 97 +--
.../migration/data/DataMigrationManager.java | 18 +-
.../data/DataMigrationManagerImpl.java | 234 ++++---
.../migration/data/MigrationDataProvider.java | 43 ++
.../core/migration/data/MigrationInfoCache.java | 52 ++
.../migration/data/MigrationInfoCacheImpl.java | 84 +++
.../data/MigrationInfoSerialization.java | 28 +-
.../data/MigrationInfoSerializationImpl.java | 67 +-
.../core/migration/data/MigrationPlugin.java | 56 ++
.../migration/data/MigrationRelationship.java | 100 +++
.../core/migration/data/PluginPhase.java | 42 ++
.../core/migration/data/ProgressObserver.java | 63 ++
.../core/migration/data/VersionedData.java | 38 ++
.../migration/data/VersionedMigrationSet.java | 153 +++++
.../core/scope/ApplicationScope.java | 1 +
.../persistence/core/task/TaskExecutor.java | 4 +-
.../MultiKeyColumnNameIteratorTest.java | 187 +++--
.../astyanax/MultiRowColumnIteratorTest.java | 50 +-
.../core/guice/DataMigrationResetRule.java | 88 +++
.../core/guice/MaxMigrationModule.java | 39 --
.../core/guice/MaxMigrationVersion.java | 40 --
.../core/guice/MigrationManagerRule.java | 7 +
.../core/guice/TestCommonModule.java | 1 +
.../persistence/core/guice/TestModule.java | 5 +-
.../data/DataMigrationManagerImplTest.java | 350 ++++++----
.../data/MigrationInfoSerializationTest.java | 32 +-
.../data/TestMigrationDataProvider.java | 61 ++
.../migration/data/TestProgressObserver.java | 89 +++
.../data/VersionedMigrationSetTest.java | 198 ++++++
.../persistence/core/util/IdGenerator.java | 51 ++
.../persistence/graph/GraphManager.java | 3 +-
.../persistence/graph/GraphManagerFactory.java | 2 +
.../persistence/graph/guice/GraphModule.java | 103 ++-
.../graph/impl/GraphManagerImpl.java | 38 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 9 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 18 +-
.../impl/stage/NodeDeleteListenerImpl.java | 18 +-
.../EdgeMetadataSerialization.java | 3 +-
.../graph/serialization/EdgesObservable.java | 33 +
.../graph/serialization/TargetIdObservable.java | 38 ++
.../EdgeMetadataSerializationProxyImpl.java | 161 +++--
.../impl/EdgeMetadataSerializationV1Impl.java | 6 +
.../impl/EdgeMetadataSerializationV2Impl.java | 6 +
.../serialization/impl/EdgesObservableImpl.java | 80 +++
.../serialization/impl/GraphDataVersions.java | 43 ++
.../impl/GraphManagerFactoryImpl.java | 95 +++
.../impl/NodeSerializationImpl.java | 36 +-
.../impl/TargetIdObservableImpl.java | 72 ++
.../impl/migration/EdgeDataMigrationImpl.java | 138 ++++
.../impl/migration/GraphMigration.java | 53 ++
.../impl/migration/GraphMigrationPlugin.java | 69 ++
.../serialization/impl/migration/GraphNode.java | 39 ++
.../shard/count/NodeShardApproximationImpl.java | 4 +-
.../NodeShardCounterSerializationImpl.java | 25 +-
.../shard/impl/NodeShardAllocationImpl.java | 16 +-
.../shard/impl/ShardGroupCompactionImpl.java | 64 +-
.../persistence/graph/GraphManagerIT.java | 38 +-
.../persistence/graph/GraphManagerLoadTest.java | 13 +-
.../graph/GraphManagerShardConsistencyIT.java | 12 +-
.../graph/GraphManagerShardingIT.java | 13 +-
.../graph/GraphManagerStressTest.java | 11 +-
.../usergrid/persistence/graph/SimpleTest.java | 12 +-
.../graph/guice/TestGraphModule.java | 27 +-
.../graph/impl/EdgeDeleteListenerTest.java | 11 +-
.../graph/impl/NodeDeleteListenerTest.java | 9 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 7 +-
.../graph/impl/stage/EdgeMetaRepairTest.java | 29 +-
.../EdgeMetaDataSerializationProxyV1Test.java | 14 +-
.../EdgeMetaDataSerializationProxyV2Test.java | 21 +-
.../EdgeMetaDataSerializationV1Test.java | 7 +-
.../EdgeMetaDataSerializationV2Test.java | 7 +-
.../EdgeMetadataSerializationTest.java | 39 +-
.../EdgeSerializationChopTest.java | 7 +-
.../serialization/EdgeSerializationTest.java | 21 +-
.../serialization/NodeSerializationTest.java | 15 +-
.../migration/EdgeDataMigrationImplTest.java | 177 +++++
.../impl/shard/EdgeShardSerializationTest.java | 7 +-
.../impl/shard/NodeShardAllocationTest.java | 27 +-
.../impl/shard/NodeShardCacheTest.java | 7 +-
.../impl/shard/ShardGroupCompactionTest.java | 7 +-
.../shard/count/NodeShardApproximationTest.java | 9 +-
.../NodeShardCounterSerializationTest.java | 6 +-
.../shard/impl/ShardEntryGroupIteratorTest.java | 15 +-
...rceDirectedEdgeDescendingComparatorTest.java | 23 +-
...getDirectedEdgeDescendingComparatorTest.java | 23 +-
.../graph/test/util/EdgeTestUtils.java | 29 +-
.../usergrid/persistence/map/MapManager.java | 25 +-
.../persistence/map/guice/MapModule.java | 8 +-
.../map/impl/MapManagerFactoryImpl.java | 29 +-
.../persistence/map/guice/TestMapModule.java | 3 +-
stack/corepersistence/model/pom.xml | 1 -
.../persistence/model/entity/Entity.java | 23 +-
.../persistence/model/entity/EntityMap.java | 66 ++
.../model/entity/EntityToMapConverter.java | 133 ++++
.../model/entity/MapToEntityConverter.java | 203 ++++++
.../persistence/model/entity/SimpleId.java | 1 -
.../persistence/model/field/AbstractField.java | 8 +
.../persistence/model/field/ListField.java | 2 +-
.../persistence/model/field/StringField.java | 20 +
.../persistence/model/util/EntityUtils.java | 72 ++
stack/corepersistence/pom.xml | 8 +-
stack/corepersistence/queryindex/pom.xml | 63 +-
.../persistence/index/AliasedEntityIndex.java | 7 +-
.../index/ApplicationEntityIndex.java | 50 ++
.../usergrid/persistence/index/EntityIndex.java | 50 +-
.../persistence/index/EntityIndexBatch.java | 5 +
.../persistence/index/EntityIndexFactory.java | 2 +-
.../persistence/index/IndexIdentifier.java | 26 +-
.../usergrid/persistence/index/SearchType.java | 49 ++
.../usergrid/persistence/index/SearchTypes.java | 14 +-
.../persistence/index/guice/IndexModule.java | 5 +-
.../impl/EsApplicationEntityIndexImpl.java | 371 ++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 80 ++-
.../index/impl/EsEntityIndexFactoryImpl.java | 27 +-
.../index/impl/EsEntityIndexImpl.java | 456 +++----------
.../persistence/index/impl/EsQueryVistor.java | 88 +--
.../persistence/index/impl/IndexingUtils.java | 140 ++--
.../persistence/index/query/EntityResults.java | 108 ---
.../usergrid/persistence/index/query/Query.java | 6 +-
.../persistence/index/query/Results.java | 148 ----
.../persistence/index/utils/ListUtils.java | 6 +-
.../persistence/index/guice/IndexTestFig.java | 62 ++
.../index/guice/TestIndexModule.java | 11 +-
.../index/impl/CorePerformanceIT.java | 339 ---------
.../impl/EntityConnectionIndexImplTest.java | 306 ---------
.../index/impl/EntityIndexMapUtils.java | 52 +-
.../persistence/index/impl/EntityIndexTest.java | 199 ++----
.../index/impl/IndexLoadTestsIT.java | 378 +++++++++++
.../queue/guice/TestQueueModule.java | 3 +-
stack/pom.xml | 16 +-
stack/rest/pom.xml | 1 +
.../usergrid/rest/AbstractContextResource.java | 102 +--
.../org/apache/usergrid/rest/IndexResource.java | 5 +-
.../apache/usergrid/rest/MigrateResource.java | 94 ++-
.../rest/test/RefreshIndexResource.java | 17 +-
.../resources/usergrid-custom-test.properties | 2 +
stack/services/pom.xml | 1 +
.../management/importer/ImportServiceImpl.java | 34 +-
.../impl/ApplicationQueueManagerImpl.java | 195 +++---
.../org/apache/usergrid/ServiceApplication.java | 18 +-
.../apache/usergrid/management/EmailFlowIT.java | 6 +-
.../usergrid/management/OrganizationIT.java | 10 +-
.../org/apache/usergrid/management/RoleIT.java | 4 +-
.../cassandra/ManagementServiceIT.java | 22 +-
.../management/export/ExportServiceIT.java | 12 +-
.../management/importer/ImportCollectionIT.java | 15 +-
.../usergrid/services/ConnectionsServiceIT.java | 10 +-
.../usergrid/services/GroupServiceIT.java | 6 +-
.../usergrid/services/RolesServiceIT.java | 2 +-
.../usergrid/services/UsersServiceIT.java | 2 +-
.../AbstractServiceNotificationIT.java | 2 +-
.../apns/NotificationsServiceIT.java | 40 +-
.../gcm/NotificationsServiceIT.java | 4 +-
.../resources/usergrid-custom-test.properties | 2 +
.../setup/ConcurrentProcessSingleton.java | 16 +-
322 files changed, 11294 insertions(+), 7800 deletions(-)
----------------------------------------------------------------------
[3/5] incubator-usergrid git commit: WIP and a mess, squash later.
Posted by to...@apache.org.
WIP and a mess, squash later.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c4f65484
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c4f65484
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c4f65484
Branch: refs/heads/USERGRID-528
Commit: c4f654847b8041a94176996feffa4060dd638819
Parents: 4c59f09
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 20 19:10:58 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 20 19:10:58 2015 -0600
----------------------------------------------------------------------
.../corepersistence/io/read/CommandBuilder.java | 36 +++---
.../io/read/EntityIndexCommands.java | 119 +++++++++++++++++++
.../corepersistence/io/state/CursorCache.java | 34 ++++++
.../corepersistence/rx/impl/CollectUntil.java | 60 ++++++++++
.../java/org/apache/usergrid/TempExample.java | 74 ++++++++++++
.../rx/impl/CollectUntilTest.java | 62 ++++++++++
6 files changed, 368 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
index 119fc0e..828cbc9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
@@ -24,6 +24,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.usergrid.corepersistence.io.reduce.StreamReducer;
+import org.apache.usergrid.corepersistence.io.state.CursorCache;
import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
@@ -35,31 +36,32 @@ import rx.functions.Func1;
*/
public class CommandBuilder {
- private final Id root;
- private final List<Command<Id>> commandList;
+ private CursorCache cache;
+ private final Observable<Id> pathObservable;
- public CommandBuilder( final Id root ) {this.root = root;
- commandList = new ArrayList<>( );
- }
- public void addIntermediateCommand(final Command<Id> command){
- commandList.add( command );
+ public CommandBuilder( final Id root ) {
+ pathObservable = Observable.just( root );
}
- public <T> void addFinalCommand( final Command<T> command, final StreamReducer<T> reducer ) {
-
- Observable.just("foo").flatMap( new Func1<String, Observable<?>>() {
- @Override
- public Observable<?> call( final String s ) {
- return null;
- }
- };
+ /**
+ * Set our cache
+ * @param cache
+ */
+ public void setCache(final CursorCache cache){
+ this.cache = cache;
}
- public List<Command<Id>> getCommands(){
- return commandList;
+ /**
+ * Returns the observable that contains the current traversal operations
+ * @return
+ */
+ public Observable<Id> getPathObservable(){
+ return pathObservable;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
new file mode 100644
index 0000000..d6f4e93
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+import rx.functions.Action2;
+import rx.functions.Func1;
+
+
+public class EntityIndexCommands {
+
+
+// /**
+// * Perform a search of all the entities, and then return the observable of search results
+// * @param index
+// * @param edgeType
+// * @param types
+// * @param query
+// * @return
+// */
+// public static Func1<Id, SearchResults> searchEntities(final ApplicationEntityIndex index,final String edgeType, final SearchTypes types, final String query ){
+//
+// return nodeId -> {
+//
+// }
+// }
+ /**
+ * Construct an indexScope from the input id type
+ * @param type
+ * @return
+ */
+ public static Func1<Id, IndexScope> createSearchScope(final String type){
+ return id -> new IndexScopeImpl( id, type );
+ }
+ /**
+ * Get our candidate results
+ * @param index
+ * @param types The types to return
+ * @param query
+ * @return
+ */
+ public static Func1<IndexScope, CandidateResults> getCandidates(final ApplicationEntityIndex index, final SearchTypes types, final String query){
+ return indexScope -> index.search( indexScope, types, Query.fromQLNullSafe( query ) );
+ }
+
+
+ /**
+ * Flattens candidate results into a single stream of a result
+ * @return
+ */
+ public static Func1<CandidateResults, Observable<CandidateResult>> flattenCandidates(){
+ return (CandidateResults candidateResults) -> Observable.from( candidateResults );
+ }
+
+
+ public static Action2<SearchResults, EntitySet> collectSet(){
+ return (searchResults, entitySet) -> {
+ searchResults.addEntities( entitySet.entities );
+ };
+ }
+
+
+
+ public static class SearchResults{
+ private final List<Entity> entities;
+ private String cursor;
+
+
+ public SearchResults(final int maxSize) {entities = new ArrayList<>(maxSize);}
+
+ public void addEntities(final Collection<Entity> entities){
+ this.entities.addAll( entities );
+
+ }
+ public void setCursor(final String cursor){
+ this.cursor = cursor;
+ }
+ }
+
+ public static class EntitySet{
+ private final List<Entity> entities;
+
+
+ public EntitySet( final List<Entity> entities ) {this.entities = entities;}
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java
new file mode 100644
index 0000000..8a5ca49
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/state/CursorCache.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.state;
+
+
+/**
+ * A class that represents a cursor cache
+ */
+public class CursorCache {
+
+ public CursorCache(){
+
+ }
+
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
new file mode 100644
index 0000000..a2cb754
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import rx.Observable;
+import rx.functions.Action2;
+import rx.functions.Func0;
+import rx.functions.Func1;
+import rx.functions.Func2;
+import rx.internal.operators.OperatorScan;
+
+
+/**
+ * An operation for performing a collect until the predicate returns true
+ */
+public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
+
+ final Func0<R> stateFactory;
+ final Action2<R, ? super T> collector;
+ final Func1<R, Boolean> predicate;
+
+
+ public CollectUntil( final Func0<R> stateFactory, final Action2<R, ? super T> collector,
+ final Func1<R, Boolean> predicate ) {
+ this.stateFactory = stateFactory;
+ this.collector = collector;
+ this.predicate = predicate;
+ }
+
+
+ @Override
+ public Observable<R> call( final Observable<T> tObservable ) {
+ Func2<R, T, R> accumulator = ( state, value ) -> {
+ collector.call( state, value );
+ return state;
+ };
+
+
+ return tObservable.lift( new OperatorScan<>( stateFactory, accumulator ) ).takeUntil( predicate );
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/test/java/org/apache/usergrid/TempExample.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/TempExample.java b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
new file mode 100644
index 0000000..1db450d
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid;
+
+
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import rx.Observable;
+
+import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.createSearchScope;
+import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.getCandidates;
+import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+
+
+public class TempExample {
+
+ //set our root observable
+
+
+ public static void main(String[] args) {
+
+ final Id rootId = createId( "thing" );
+
+ final ApplicationEntityIndex index = null;
+
+
+ final SearchTypes searchType = SearchTypes.fromTypes( "test" );
+
+ final String query = "select * ";
+
+ final Observable<CandidateResults> observable = Observable.just( rootId ).map( createSearchScope( "type" ) ).map(getCandidates(index, searchType, query));
+
+
+ observable.doOnNext( a -> System.out.println( a) ).toBlocking().last();
+ }
+
+
+ private static final class ResultsCollector{
+
+ /**
+ * Add the candidates to our collection
+ * @param results
+ */
+ public void addCandidates(final CandidateResults results ){
+
+ //TODO, collect the results, removing groups
+ Observable.from( results ).groupBy( candidate -> candidate.getId() ).collect( )
+ }
+
+ }
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c4f65484/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
new file mode 100644
index 0000000..ce12429
--- /dev/null
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.rx.impl;
+
+
+import org.junit.Test;
+
+import rx.Observable;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class CollectUntilTest {
+
+ @Test
+ public void testCollectUntil() {
+
+ final CollectUntil<Integer, CountCollector> collectUntil =
+ new CollectUntil<>(
+ () -> new CountCollector(),
+ ( collector, value ) -> collector.mark(),
+ collector -> collector.isFull() );
+
+
+ final CountCollector collector = Observable.range( 0, 200 ).compose( collectUntil ).toBlocking().last();
+
+ assertEquals( 100, collector.count );
+ }
+
+
+ private static final class CountCollector {
+
+ private int count;
+
+
+ public void mark() {
+ count++;
+ }
+
+
+ public boolean isFull() {
+ return count == 100;
+ }
+ }
+}
[5/5] incubator-usergrid git commit: WIP, overwrite
Posted by to...@apache.org.
WIP, overwrite
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cee76da7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cee76da7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cee76da7
Branch: refs/heads/USERGRID-528
Commit: cee76da72c3343d562867988f76f264204d54b59
Parents: 61aa979
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Mar 24 14:15:49 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Mar 24 14:15:49 2015 -0600
----------------------------------------------------------------------
.../io/read/EntityIndexCommand.java | 404 ++++++++++++-------
.../corepersistence/rx/impl/CollectUntil.java | 2 +-
2 files changed, 270 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cee76da7/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
index a8fa221..464dd45 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
@@ -22,8 +22,9 @@ package org.apache.usergrid.corepersistence.io.read;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.collection.CollectionScope;
@@ -32,8 +33,8 @@ import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.IndexScope;
import org.apache.usergrid.persistence.index.SearchTypes;
import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
import org.apache.usergrid.persistence.index.query.CandidateResult;
@@ -52,32 +53,34 @@ import rx.functions.Func0;
import rx.functions.Func1;
+/**
+ * Performs a search of the given type along the specified edgeName. It then loads and validates results,
+ * and returns an Observable of SearchResults
+ */
@Singleton
public class EntityIndexCommand implements Command<Id, EntityIndexCommand.SearchResults> {
private final Id applicationId;
- private final ApplicationScope applicationScope;
private final ApplicationEntityIndex index;
private final SearchTypes types;
private final String query;
private final int resultSetSize;
- private final String scopeType;
+ private final String edgeName;
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
@Inject
- public EntityIndexCommand( final Id applicationId, final ApplicationScope applicationScope,
- final ApplicationEntityIndex index, final SearchTypes types, final String query,
- final int resultSetSize, final String scopeType,
+ public EntityIndexCommand( final Id applicationId, final ApplicationEntityIndex index, final SearchTypes types,
+ final String query, final int resultSetSize, final String edgeName,
final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
this.applicationId = applicationId;
- this.applicationScope = applicationScope;
+
this.index = index;
this.types = types;
this.query = query;
this.resultSetSize = resultSetSize;
- this.scopeType = scopeType;
+ this.edgeName = edgeName;
this.entityCollectionManagerFactory = entityCollectionManagerFactory;
}
@@ -85,150 +88,216 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
@Override
public Observable<EntityIndexCommand.SearchResults> call( final Observable<Id> idObservable ) {
- //create our observable of candidate search results
- final Observable<CandidateResults> candidateResults = idObservable
- .flatMap( id -> Observable.create( new ElasticSearchObservable( initialSearch( id ), nextPage( id ) ) ) );
-
- final Observable<CandidateResult> candidateObservable =
- candidateResults.flatMap( candidates -> Observable.from( candidates ) );
-
- //since we'll only have at most 100 results in memory at a time, we roll up our groups and emit them on to
- // the collector
- final Observable<CandidateGroup> candidateGroup =
- candidateObservable.groupBy( candidate -> candidate.getId() ).map( observableGroup -> {
-
-
- //for each group, create a list, then sort by highest version first
- final List<CandidateResult> groupList = observableGroup.toList().toBlocking().last();
-
- Collections.sort( groupList, CandidateVersionComparator::compare );
-
- //create our candidate group and emit it
- final CandidateGroup group =
- new CandidateGroup( groupList.get( 0 ), groupList.subList( 1, groupList.size() ) );
-
- return group;
- } );
-
+ //load all the potential candidates
+ final Observable<CandidateCollector> collectedCandidates = idObservable.compose( createCandidates() );
- //buffer our candidate group up to our resultset size.
- final Observable<CandidateCollector> collectedCandidates =
- candidateGroup.buffer( resultSetSize ).flatMap( candidates -> {
+ //now we have our collected candidates, load them from ES
+ final Observable<SearchResults> loadedEntities = collectedCandidates.map( loadEntities( resultSetSize ) );
- final Observable<CandidateCollector> collector = Observable.from( candidates ).collect(
- () -> new CandidateCollector( resultSetSize ), ( candidateCollector, candidate ) -> {
- //add our candidates to our collector
- candidateCollector.addCandidate( candidate.toKeep );
- //add our empty results
- candidateCollector.addEmptyResults( candidate.toRemove );
- } );
-
- return collector;
- } );
- //now we have our collected candidates, load them
+ //after we've loaded, remove everything that's left
+ loadedEntities.doOnNext( results ->{
+ //run the deindex
+ Observable.from( results.toRemove ).collect( () -> index.createBatch(), (batch, toRemove) -> {
+ batch.deindex( toRemove.indexScope, toRemove.result );
+ }).doOnNext( batch -> batch.execute() ).toBlocking().lastOrDefault( null );
- final Observable<SearchResults> loadedEntities = collectedCandidates.map( loadEntities(resultSetSize) );
+ });
return loadedEntities;
}
- /**
- * Perform the initial search with the sourceId
- */
- private Func0<CandidateResults> initialSearch( final Id sourceId ) {
- return () -> index.search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ) );
- }
/**
- * Search the next page for the specified source
+ * Using an input of observable ids, transform and collect them into a set of CandidateCollector to be loaded, or
+ * verified by the EM
*/
- private Func1<String, CandidateResults> nextPage( final Id sourceId ) {
- return cursor -> index
- .search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ).withCursor( cursor ) );
- }
+ private Observable.Transformer<Id, CandidateCollector> createCandidates() {
+ return idObservable -> {
+ //create our observable of candidate search results
+ final Observable<ScopedCandidateResults> candidateResults = idObservable.flatMap(
- /**
- * Function that will load our entities from our candidates, filter stale or missing candidates and return results
- * @param expectedSize
- * @return
- */
- private Func1<CandidateCollector, SearchResults> loadEntities(final int expectedSize) {
- return candidateCollector -> {
- //from our candidates, group them by id type so we can create scopes
- Observable.from( candidateCollector.getCandidates() ).groupBy( candidate -> candidate.getId().getType() )
- .flatMap( groups -> {
+ id -> {
+ //create the index scope from the id
+ final IndexScope scope = createScope( id );
- final List<CandidateResult> candidates = groups.toList().toBlocking().last();
+ //perform the ES search of candidate results
+ final Observable<ScopedCandidateResults> scoped = Observable.create(
+ new ElasticSearchObservable( initialSearch( scope ), nextPage( scope ) ) );
- //we can get no results, so quit aggregating if there are none
- if ( candidates.size() == 0 ) {
- return Observable.just( new SearchResults( 0 ) );
- }
+ final Observable<CandidateCollector> collectedResults = scoped.flatMap( scopedCandidateResults -> {
- final String typeName = candidates.get( 0 ).getId().getType();
+ //here b/c the compiler cannot infer type directly
+ final Observable<CandidateResult> candidates =
+ Observable.from( scopedCandidateResults.candidateResults );
- final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType( typeName );
+ return candidates.collect( () -> new CandidateCollector( resultSetSize,
+ scopedCandidateResults.candidateResults.getCursor() ), ( collector, candidate ) -> {
+ collector.insert( candidate );
+ } );
+ } );
- //create our scope to get our entity manager
- final CollectionScope scope =
- new CollectionScopeImpl( applicationId, applicationId, collectionType );
+ //group our candidates by type
+// collectedResults.flatMap( collected -> Observable.from( collected.candidates ) ).groupBy( candidate -> candidate.getId().getType() ).co
- final EntityCollectionManager ecm =
- entityCollectionManagerFactory.createCollectionManager( scope );
+ final Observable<SearchResults> resultsObservable = collectedResults.map( collectedCandidates -> {
- //get our entity ids
- final List<Id> entityIds =
- Observable.from( candidates ).map( c -> c.getId() ).toList().toBlocking().last();
- //TODO, change this out
+ //we can get no results, so quit aggregating if there are none
+ if ( collectedCandidates.candidates.size() == 0 ) {
+ return new SearchResults( 0, collectedCandidates.cursor );
+ }
- //an observable of our entity set
+ final String collectionType =
+ CpNamingUtils.getCollectionScopeNameFromEntityType( id.getType() );
- //now go through all our candidates and verify
+ //create our scope to get our entity manager
- return Observable.from( candidates ).collect( () -> new SearchResults( expectedSize ), (searchResults, candidate) ->{
+ final CollectionScope collectionScope =
+ new CollectionScopeImpl( applicationId, applicationId, collectionType );
- final EntitySet entitySet = ecm.load( entityIds ).toBlocking().last();
+ final EntityCollectionManager ecm =
+ entityCollectionManagerFactory.createCollectionManager( collectionScope );
- final MvccEntity entity = entitySet.getEntity( candidate.getId() );
+ //get our entity ids
+ final List<Id> entityIds =
+ Observable.from( collectedCandidates.candidates ).map( c -> c.getId() ).toList()
+ .toBlocking().last();
- //our candidate it stale, or target entity was deleted add it to the remove of our collector
- if(UUIDComparator.staticCompare( entity.getVersion(), candidate.getVersion()) > 0 || !entity.getEntity().isPresent()){
- searchResults.addToRemove( candidate );
- return;
- }
+ //TODO, change this out
+ //an observable of our entity set
- searchResults.addEntity( entity.getEntity().get() );
+ return ecm.load( entityIds ).map( results -> {
+ Observable.from( entityIds ).collect( () -> new SearchResults( resultSetSize ),
+ ( searchResults, entityIds ) -> {
+ } );
+ } );
+ } );
- } )
- //add the existing set to remove to this set
- .doOnNext( results -> results.addToRemove( candidateCollector.getToRemove() ) );
+ return resultsObservable;
+ } );
- } );
+ return candidateResults;
+ };
+ }
- return null;
- };
+ /**
+ * Perform the initial search with the sourceId
+ */
+ private Func0<ScopedCandidateResults> initialSearch( final IndexScope scope ) {
+ return () -> new ScopedCandidateResults(scope, index.search( scope, types, Query.fromQL( query ) ) );
+ }
+
+
+ /**
+ * Search the next page for the specified source
+ */
+ private Func1<String, ScopedCandidateResults> nextPage( final IndexScope scope ) {
+ return cursor -> new ScopedCandidateResults(scope, index.search( scope, types, Query.fromQL( query ).withCursor(
+ cursor ) ) );
}
+ /**
+ * Create the scope and return it
+ * @param sourceId
+ * @return
+ */
+ private IndexScope createScope(final Id sourceId){
+ return new IndexScopeImpl( sourceId, edgeName );
+ }
+
+//
+// /**
+// * Function that will load our entities from our candidates, filter stale or missing candidates and return results
+// */
+// private Func1<CandidateCollector, SearchResults> loadEntities( final int expectedSize ) {
+// return candidateCollector -> {
+//
+// //from our candidates, group them by id type so we can create scopes
+// Observable.from( candidateCollector.getCandidates() ).groupBy( candidate -> candidate.result.getId().getType() )
+// .flatMap( groups -> {
+//
+//
+// final List<ScopedCandidateResult> candidates = groups.toList().toBlocking().last();
+//
+// //we can get no results, so quit aggregating if there are none
+// if ( candidates.size() == 0 ) {
+// return Observable.just( new SearchResults( 0, null ) );
+// }
+//
+//
+// final String typeName = candidates.get( 0 ).result.getId().getType();
+//
+// final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType( typeName );
+//
+//
+// //create our scope to get our entity manager
+//
+// final CollectionScope scope =
+// new CollectionScopeImpl( applicationId, applicationId, collectionType );
+//
+// final EntityCollectionManager ecm =
+// entityCollectionManagerFactory.createCollectionManager( scope );
+//
+//
+// //get our entity ids
+// final List<Id> entityIds =
+// Observable.from( candidates ).map( c -> c.result.getId() ).toList().toBlocking().last();
+//
+// //TODO, change this out
+//
+// //an observable of our entity set
+//
+//
+// //now go through all our candidates and verify
+//
+// return Observable.from( candidates ).collect( () -> new SearchResults(
+// expectedSize ),
+// ( searchResults, candidate ) -> {
+//
+// final EntitySet entitySet = ecm.load( entityIds ).toBlocking().last();
+//
+// final MvccEntity entity = entitySet.getEntity( candidate.result.getId() );
+//
+//
+// //our candidate it stale, or target entity was deleted add it to the remove of our
+// // collector
+// if ( UUIDComparator.staticCompare( entity.getVersion(), candidate.result.getVersion() ) > 0
+// || !entity.getEntity().isPresent() ) {
+//
+// searchResults.addToRemove( candidate );
+// return;
+// }
+//
+//
+// searchResults.addEntity( entity.getEntity().get() );
+// } )
+// //add the existing set to remove to this set
+// .doOnNext( results -> results.addToRemove( candidateCollector.getToRemove() ) );
+// } );
+//
+//
+// return null;
+// };
+// }
/**
@@ -238,20 +307,49 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
private final List<CandidateResult> candidates;
private final List<CandidateResult> toRemove;
+ private final Map<Id, Integer> indexMapping;
- public CandidateCollector( final int maxSize ) {
+ private final String cursor;
+
+
+ public CandidateCollector( final int maxSize, final String cursor ) {
+ this.cursor = cursor;
candidates = new ArrayList<>( maxSize );
toRemove = new ArrayList<>( maxSize );
+ indexMapping = new HashMap<>(maxSize);
}
- public void addCandidate( final CandidateResult candidate ) {
- this.candidates.add( candidate );
- }
+ public void insert(final CandidateResult newValue){
+
+ final Id candidateId = newValue.getId();
+
+ final Integer index = indexMapping.get( candidateId );
+
+ if(index == null){
+ candidates.add( newValue );
+ indexMapping.put( candidateId, candidates.size()-1);
+ return;
+ }
+
+ //present, perform a comparison
+
+ final CandidateResult existing = candidates.get( index );
+
+
+ //it's a greater version, add this to ignore
+ if(UUIDComparator.staticCompare( existing.getVersion(), newValue.getVersion() ) > 0){
+ toRemove.add( newValue );
+ }
+
+ //remove the stale version from the list and put it in deindex
+ else{
+ candidates.remove( index );
+ candidates.add( newValue );
+ toRemove.add( existing );
+ }
- public void addEmptyResults( final Collection<CandidateResult> stale ) {
- this.toRemove.addAll( stale );
}
@@ -268,13 +366,16 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
public static class SearchResults {
private final List<Entity> entities;
- private final List<CandidateResult> toRemove;
+ private final List<ScopedCandidateResult> toRemove;
+ private final int maxSize;
- private String cursor;
+ private final String cursor;
- public SearchResults( final int maxSize ) {
- entities = new ArrayList<>( maxSize );
+ public SearchResults( final int maxSize, final String cursor ) {
+ this.maxSize = maxSize;
+ this.cursor = cursor;
+ this.entities = new ArrayList<>( maxSize );
this.toRemove = new ArrayList<>( maxSize );
}
@@ -284,20 +385,23 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
}
- public void addToRemove( final Collection<CandidateResult> stale ) {
+ public void addToRemove( final Collection<ScopedCandidateResult> stale ) {
this.toRemove.addAll( stale );
}
- public void addToRemove( final CandidateResult candidateResult ) {
- this.toRemove.add( candidateResult );
- }
+ public void addToRemove( final ScopedCandidateResult candidateResult ) {
+ this.toRemove.add( candidateResult );
+ }
+ public String getCursor() {
+ return cursor;
+ }
- public void setCursor( final String cursor ) {
- this.cursor = cursor;
+ public boolean isFull(){
+ return entities.size() >= maxSize;
}
}
@@ -305,26 +409,26 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
/**
* An observable that will perform a search and continually emit results while they exist.
*/
- public static class ElasticSearchObservable implements Observable.OnSubscribe<CandidateResults> {
+ public static class ElasticSearchObservable implements Observable.OnSubscribe<ScopedCandidateResults> {
- private final Func1<String, CandidateResults> fetchNextPage;
- private final Func0<CandidateResults> fetchInitialResults;
+ private final Func1<String, ScopedCandidateResults> fetchNextPage;
+ private final Func0<ScopedCandidateResults> fetchInitialResults;
- public ElasticSearchObservable( final Func0<CandidateResults> fetchInitialResults,
- final Func1<String, CandidateResults> fetchNextPage ) {
+ public ElasticSearchObservable( final Func0<ScopedCandidateResults> fetchInitialResults,
+ final Func1<String, ScopedCandidateResults> fetchNextPage ) {
this.fetchInitialResults = fetchInitialResults;
this.fetchNextPage = fetchNextPage;
}
@Override
- public void call( final Subscriber<? super CandidateResults> subscriber ) {
+ public void call( final Subscriber<? super ScopedCandidateResults> subscriber ) {
subscriber.onStart();
try {
- CandidateResults results = fetchInitialResults.call();
+ ScopedCandidateResults results = fetchInitialResults.call();
//emit our next page
@@ -332,13 +436,13 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
subscriber.onNext( results );
//if we have no cursor, we're done
- if ( !results.hasCursor() ) {
+ if ( !results.candidateResults.hasCursor() ) {
break;
}
//we have a cursor, get our results to emit for the next page
- results = fetchNextPage.call( results.getCursor() );
+ results = fetchNextPage.call( results.candidateResults.getCursor() );
}
subscriber.onCompleted();
@@ -351,16 +455,51 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
/**
- * A message that contains the candidate to keep, and the candidate toRemove
+ * A candidate result, aslong with the scope it was returned in
+ */
+ public static class ScopedCandidateResult{
+ private final IndexScope indexScope;
+ private final CandidateResult result;
+
+
+ public ScopedCandidateResult( final IndexScope indexScope, final CandidateResult result ) {
+ this.indexScope = indexScope;
+ this.result = result;
+ }
+ }
+
+
+ /**
+ * Object that represents our candidate results, along with the scope it was searched in
+ */
+ public static class ScopedCandidateResults {
+ private final IndexScope indexScope;
+ private CandidateResults candidateResults;
+
+
+ public ScopedCandidateResults( final IndexScope indexScope, final CandidateResults candidateResults ) {
+ this.indexScope = indexScope;
+ this.candidateResults = candidateResults;
+ }
+ }
+
+
+ /**
+ * A message that contains the candidate to keep, and the candidate toRemove and the scope they were searched in
*/
public static class CandidateGroup {
+ private final IndexScope indexScope;
private final CandidateResult toKeep;
private final Collection<CandidateResult> toRemove;
+ private final String cursor;
- public CandidateGroup( final CandidateResult toKeep, final Collection<CandidateResult> toRemove ) {
+ public CandidateGroup( final IndexScope indexScope, final CandidateResult toKeep,
+ final Collection<CandidateResult> toRemove, final String cursor ) {
+ this.indexScope = indexScope;
this.toKeep = toKeep;
this.toRemove = toRemove;
+ this.cursor = cursor;
}
}
@@ -375,10 +514,5 @@ public class EntityIndexCommand implements Command<Id, EntityIndexCommand.Search
}
}
- /***********************
- * FROM HERE DOWN IS EXPERIMENTAL
- *************************
- */
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cee76da7/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
index 8a0c014..93de094 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
@@ -38,7 +38,7 @@ public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
final Func1<R, Boolean> shortCircuitWhen;
- public CollectUntil( final Func1<R, Boolean> shortCircuitWhen, final Func0<R> stateFactory, final Action2<R, ? super T> collector) {
+ public CollectUntil( final Func0<R> stateFactory,final Func1<R, Boolean> shortCircuitWhen, final Action2<R, ? super T> collector) {
this.stateFactory = stateFactory;
this.collector = collector;
this.shortCircuitWhen = shortCircuitWhen;
[4/5] incubator-usergrid git commit: Refactor to clean up. Still a WIP
Posted by to...@apache.org.
Refactor to clean up. Still a WIP
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/61aa9794
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/61aa9794
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/61aa9794
Branch: refs/heads/USERGRID-528
Commit: 61aa97945313533f6ef676f3f9dab5a22b9cb63b
Parents: c4f6548
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 23 09:52:44 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 23 17:23:46 2015 -0600
----------------------------------------------------------------------
.../corepersistence/io/read/Command.java | 8 +-
.../corepersistence/io/read/CommandBuilder.java | 8 +-
.../io/read/EntityIndexCommand.java | 384 +++++++++++++++++++
.../io/read/EntityIndexCommands.java | 119 ------
.../corepersistence/rx/impl/CollectUntil.java | 11 +-
.../java/org/apache/usergrid/TempExample.java | 217 ++++++++++-
.../rx/impl/CollectUntilTest.java | 6 +-
7 files changed, 605 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
index 4e3b0fc..34b8d1e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/Command.java
@@ -25,14 +25,8 @@ import org.apache.usergrid.persistence.model.entity.Id;
import rx.Observable;
-public interface Command<T> {
+public interface Command<T, R> extends Observable.Transformer<T, R> {
- /**
- * Process our input stream
- * @param input
- * @return
- */
- public Observable<T> process( Observable<T> input );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
index 828cbc9..bbf74b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/CommandBuilder.java
@@ -38,7 +38,7 @@ public class CommandBuilder {
private CursorCache cache;
- private final Observable<Id> pathObservable;
+ private Observable<Id> pathObservable;
public CommandBuilder( final Id root ) {
@@ -46,6 +46,12 @@ public class CommandBuilder {
}
+
+ public void addCommand(final Command<Id, Id> intermediateCommand){
+ pathObservable = pathObservable.compose( intermediateCommand );
+ }
+
+
/**
* Set our cache
* @param cache
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
new file mode 100644
index 0000000..a8fa221
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommand.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence.io.read;
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+import rx.Observable;
+import rx.Subscriber;
+import rx.functions.Func0;
+import rx.functions.Func1;
+
+
+@Singleton
+public class EntityIndexCommand implements Command<Id, EntityIndexCommand.SearchResults> {
+
+ private final Id applicationId;
+ private final ApplicationScope applicationScope;
+ private final ApplicationEntityIndex index;
+ private final SearchTypes types;
+ private final String query;
+ private final int resultSetSize;
+ private final String scopeType;
+ private final EntityCollectionManagerFactory entityCollectionManagerFactory;
+
+
+ @Inject
+ public EntityIndexCommand( final Id applicationId, final ApplicationScope applicationScope,
+ final ApplicationEntityIndex index, final SearchTypes types, final String query,
+ final int resultSetSize, final String scopeType,
+ final EntityCollectionManagerFactory entityCollectionManagerFactory ) {
+ this.applicationId = applicationId;
+ this.applicationScope = applicationScope;
+
+ this.index = index;
+ this.types = types;
+ this.query = query;
+ this.resultSetSize = resultSetSize;
+ this.scopeType = scopeType;
+ this.entityCollectionManagerFactory = entityCollectionManagerFactory;
+ }
+
+
+ @Override
+ public Observable<EntityIndexCommand.SearchResults> call( final Observable<Id> idObservable ) {
+
+ //create our observable of candidate search results
+ final Observable<CandidateResults> candidateResults = idObservable
+ .flatMap( id -> Observable.create( new ElasticSearchObservable( initialSearch( id ), nextPage( id ) ) ) );
+
+ final Observable<CandidateResult> candidateObservable =
+ candidateResults.flatMap( candidates -> Observable.from( candidates ) );
+
+ //since we'll only have at most 100 results in memory at a time, we roll up our groups and emit them on to
+ // the collector
+ final Observable<CandidateGroup> candidateGroup =
+ candidateObservable.groupBy( candidate -> candidate.getId() ).map( observableGroup -> {
+
+
+ //for each group, create a list, then sort by highest version first
+ final List<CandidateResult> groupList = observableGroup.toList().toBlocking().last();
+
+ Collections.sort( groupList, CandidateVersionComparator::compare );
+
+ //create our candidate group and emit it
+ final CandidateGroup group =
+ new CandidateGroup( groupList.get( 0 ), groupList.subList( 1, groupList.size() ) );
+
+ return group;
+ } );
+
+
+ //buffer our candidate group up to our resultset size.
+ final Observable<CandidateCollector> collectedCandidates =
+ candidateGroup.buffer( resultSetSize ).flatMap( candidates -> {
+
+ final Observable<CandidateCollector> collector = Observable.from( candidates ).collect(
+ () -> new CandidateCollector( resultSetSize ), ( candidateCollector, candidate ) -> {
+ //add our candidates to our collector
+ candidateCollector.addCandidate( candidate.toKeep );
+ //add our empty results
+ candidateCollector.addEmptyResults( candidate.toRemove );
+ } );
+
+ return collector;
+ } );
+
+ //now we have our collected candidates, load them
+
+
+ final Observable<SearchResults> loadedEntities = collectedCandidates.map( loadEntities(resultSetSize) );
+
+
+ return loadedEntities;
+ }
+
+
+ /**
+ * Perform the initial search with the sourceId
+ */
+ private Func0<CandidateResults> initialSearch( final Id sourceId ) {
+ return () -> index.search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ) );
+ }
+
+
+ /**
+ * Search the next page for the specified source
+ */
+ private Func1<String, CandidateResults> nextPage( final Id sourceId ) {
+ return cursor -> index
+ .search( new IndexScopeImpl( sourceId, scopeType ), types, Query.fromQL( query ).withCursor( cursor ) );
+ }
+
+
+ /**
+ * Function that will load our entities from our candidates, filter stale or missing candidates and return results
+ * @param expectedSize
+ * @return
+ */
+ private Func1<CandidateCollector, SearchResults> loadEntities(final int expectedSize) {
+ return candidateCollector -> {
+
+ //from our candidates, group them by id type so we can create scopes
+ Observable.from( candidateCollector.getCandidates() ).groupBy( candidate -> candidate.getId().getType() )
+ .flatMap( groups -> {
+
+
+ final List<CandidateResult> candidates = groups.toList().toBlocking().last();
+
+ //we can get no results, so quit aggregating if there are none
+ if ( candidates.size() == 0 ) {
+ return Observable.just( new SearchResults( 0 ) );
+ }
+
+
+ final String typeName = candidates.get( 0 ).getId().getType();
+
+ final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType( typeName );
+
+
+ //create our scope to get our entity manager
+
+ final CollectionScope scope =
+ new CollectionScopeImpl( applicationId, applicationId, collectionType );
+
+ final EntityCollectionManager ecm =
+ entityCollectionManagerFactory.createCollectionManager( scope );
+
+
+ //get our entity ids
+ final List<Id> entityIds =
+ Observable.from( candidates ).map( c -> c.getId() ).toList().toBlocking().last();
+
+ //TODO, change this out
+
+ //an observable of our entity set
+
+
+
+ //now go through all our candidates and verify
+
+ return Observable.from( candidates ).collect( () -> new SearchResults( expectedSize ), (searchResults, candidate) ->{
+
+ final EntitySet entitySet = ecm.load( entityIds ).toBlocking().last();
+
+ final MvccEntity entity = entitySet.getEntity( candidate.getId() );
+
+
+ //our candidate it stale, or target entity was deleted add it to the remove of our collector
+ if(UUIDComparator.staticCompare( entity.getVersion(), candidate.getVersion()) > 0 || !entity.getEntity().isPresent()){
+ searchResults.addToRemove( candidate );
+ return;
+ }
+
+
+ searchResults.addEntity( entity.getEntity().get() );
+
+
+ } )
+ //add the existing set to remove to this set
+ .doOnNext( results -> results.addToRemove( candidateCollector.getToRemove() ) );
+
+ } );
+
+
+ return null;
+ };
+ }
+
+
+
+
+ /**
+ * Collects all valid results (in order) as well as candidates to be removed
+ */
+ public static class CandidateCollector {
+ private final List<CandidateResult> candidates;
+ private final List<CandidateResult> toRemove;
+
+
+ public CandidateCollector( final int maxSize ) {
+ candidates = new ArrayList<>( maxSize );
+ toRemove = new ArrayList<>( maxSize );
+ }
+
+
+ public void addCandidate( final CandidateResult candidate ) {
+ this.candidates.add( candidate );
+ }
+
+
+ public void addEmptyResults( final Collection<CandidateResult> stale ) {
+ this.toRemove.addAll( stale );
+ }
+
+
+ public List<CandidateResult> getCandidates() {
+ return candidates;
+ }
+
+
+ public List<CandidateResult> getToRemove() {
+ return toRemove;
+ }
+ }
+
+
+ public static class SearchResults {
+ private final List<Entity> entities;
+ private final List<CandidateResult> toRemove;
+
+ private String cursor;
+
+
+ public SearchResults( final int maxSize ) {
+ entities = new ArrayList<>( maxSize );
+ this.toRemove = new ArrayList<>( maxSize );
+ }
+
+
+ public void addEntity( final Entity entity ) {
+ this.entities.add( entity );
+ }
+
+
+ public void addToRemove( final Collection<CandidateResult> stale ) {
+ this.toRemove.addAll( stale );
+ }
+
+
+ public void addToRemove( final CandidateResult candidateResult ) {
+ this.toRemove.add( candidateResult );
+ }
+
+
+
+
+ public void setCursor( final String cursor ) {
+ this.cursor = cursor;
+ }
+ }
+
+
+ /**
+ * An observable that will perform a search and continually emit results while they exist.
+ */
+ public static class ElasticSearchObservable implements Observable.OnSubscribe<CandidateResults> {
+
+ private final Func1<String, CandidateResults> fetchNextPage;
+ private final Func0<CandidateResults> fetchInitialResults;
+
+
+ public ElasticSearchObservable( final Func0<CandidateResults> fetchInitialResults,
+ final Func1<String, CandidateResults> fetchNextPage ) {
+ this.fetchInitialResults = fetchInitialResults;
+ this.fetchNextPage = fetchNextPage;
+ }
+
+
+ @Override
+ public void call( final Subscriber<? super CandidateResults> subscriber ) {
+
+ subscriber.onStart();
+
+ try {
+ CandidateResults results = fetchInitialResults.call();
+
+
+ //emit our next page
+ while ( true ) {
+ subscriber.onNext( results );
+
+ //if we have no cursor, we're done
+ if ( !results.hasCursor() ) {
+ break;
+ }
+
+
+ //we have a cursor, get our results to emit for the next page
+ results = fetchNextPage.call( results.getCursor() );
+ }
+
+ subscriber.onCompleted();
+ }
+ catch ( Throwable t ) {
+ subscriber.onError( t );
+ }
+ }
+ }
+
+
+ /**
+ * A message that contains the candidate to keep, and the candidate toRemove
+ */
+ public static class CandidateGroup {
+ private final CandidateResult toKeep;
+ private final Collection<CandidateResult> toRemove;
+
+
+ public CandidateGroup( final CandidateResult toKeep, final Collection<CandidateResult> toRemove ) {
+ this.toKeep = toKeep;
+ this.toRemove = toRemove;
+ }
+ }
+
+
+ /**
+ * Compares 2 candidates by version. The max version is considered greater
+ */
+ private static final class CandidateVersionComparator {
+
+ public static int compare( final CandidateResult o1, final CandidateResult o2 ) {
+ return UUIDComparator.staticCompare( o1.getVersion(), o2.getVersion() );
+ }
+ }
+
+ /***********************
+ * FROM HERE DOWN IS EXPERIMENTAL
+ *************************
+ */
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
deleted file mode 100644
index d6f4e93..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/io/read/EntityIndexCommands.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.corepersistence.io.read;
-
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.SearchTypes;
-import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.index.query.Query;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import rx.Observable;
-import rx.functions.Action2;
-import rx.functions.Func1;
-
-
-public class EntityIndexCommands {
-
-
-// /**
-// * Perform a search of all the entities, and then return the observable of search results
-// * @param index
-// * @param edgeType
-// * @param types
-// * @param query
-// * @return
-// */
-// public static Func1<Id, SearchResults> searchEntities(final ApplicationEntityIndex index,final String edgeType, final SearchTypes types, final String query ){
-//
-// return nodeId -> {
-//
-// }
-// }
- /**
- * Construct an indexScope from the input id type
- * @param type
- * @return
- */
- public static Func1<Id, IndexScope> createSearchScope(final String type){
- return id -> new IndexScopeImpl( id, type );
- }
- /**
- * Get our candidate results
- * @param index
- * @param types The types to return
- * @param query
- * @return
- */
- public static Func1<IndexScope, CandidateResults> getCandidates(final ApplicationEntityIndex index, final SearchTypes types, final String query){
- return indexScope -> index.search( indexScope, types, Query.fromQLNullSafe( query ) );
- }
-
-
- /**
- * Flattens candidate results into a single stream of a result
- * @return
- */
- public static Func1<CandidateResults, Observable<CandidateResult>> flattenCandidates(){
- return (CandidateResults candidateResults) -> Observable.from( candidateResults );
- }
-
-
- public static Action2<SearchResults, EntitySet> collectSet(){
- return (searchResults, entitySet) -> {
- searchResults.addEntities( entitySet.entities );
- };
- }
-
-
-
- public static class SearchResults{
- private final List<Entity> entities;
- private String cursor;
-
-
- public SearchResults(final int maxSize) {entities = new ArrayList<>(maxSize);}
-
- public void addEntities(final Collection<Entity> entities){
- this.entities.addAll( entities );
-
- }
- public void setCursor(final String cursor){
- this.cursor = cursor;
- }
- }
-
- public static class EntitySet{
- private final List<Entity> entities;
-
-
- public EntitySet( final List<Entity> entities ) {this.entities = entities;}
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
index a2cb754..8a0c014 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntil.java
@@ -29,20 +29,19 @@ import rx.internal.operators.OperatorScan;
/**
- * An operation for performing a collect until the predicate returns true
+ * An operation for performing a collect until the shortCircuitWhen returns true
*/
public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
final Func0<R> stateFactory;
final Action2<R, ? super T> collector;
- final Func1<R, Boolean> predicate;
+ final Func1<R, Boolean> shortCircuitWhen;
- public CollectUntil( final Func0<R> stateFactory, final Action2<R, ? super T> collector,
- final Func1<R, Boolean> predicate ) {
+ public CollectUntil( final Func1<R, Boolean> shortCircuitWhen, final Func0<R> stateFactory, final Action2<R, ? super T> collector) {
this.stateFactory = stateFactory;
this.collector = collector;
- this.predicate = predicate;
+ this.shortCircuitWhen = shortCircuitWhen;
}
@@ -54,7 +53,7 @@ public class CollectUntil<T, R> implements Observable.Transformer<T, R> {
};
- return tObservable.lift( new OperatorScan<>( stateFactory, accumulator ) ).takeUntil( predicate );
+ return tObservable.lift( new OperatorScan<>( stateFactory, accumulator ) ).takeUntil( shortCircuitWhen );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/test/java/org/apache/usergrid/TempExample.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/TempExample.java b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
index 1db450d..ed58111 100644
--- a/stack/core/src/test/java/org/apache/usergrid/TempExample.java
+++ b/stack/core/src/test/java/org/apache/usergrid/TempExample.java
@@ -20,16 +20,37 @@
package org.apache.usergrid;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.junit.Test;
+
+import org.apache.usergrid.corepersistence.rx.impl.CollectUntil;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.fasterxml.uuid.UUIDComparator;
+
import rx.Observable;
+import rx.Statement;
+import rx.Subscriber;
+import rx.functions.Action2;
+import rx.functions.Func0;
+import rx.functions.Func1;
+import rx.observables.GroupedObservable;
import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.createSearchScope;
import static org.apache.usergrid.corepersistence.io.read.EntityIndexCommands.getCandidates;
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
+import static org.junit.Assert.assertEquals;
public class TempExample {
@@ -37,38 +58,210 @@ public class TempExample {
//set our root observable
- public static void main(String[] args) {
- final Id rootId = createId( "thing" );
+//
+// final Id rootId = createId( "thing" );
+//
+// final ApplicationEntityIndex index = null;
+//
+//
+// final SearchTypes searchType = SearchTypes.fromTypes( "test" );
+//
+// final String query = "select * ";
+//
+// final int selectSize = 100;
+//
+// final Observable<CandidateResults> observable = Observable.just( rootId ).map( createSearchScope( "type" ) )
+// .map( getCandidates( index, searchType,
+// query ) ); //.compose();
+//
+//
+// final Observable<GroupedObservable<Id, CandidateResult>> next =
+// observable.flatMap( candidates -> Observable.from( candidates ) ).groupBy( result -> result.getId() );
+//
+//
+// final Observable<ResultsCollector> results = next.compose( new CollectUntil<>(
+// //stop when the collector has a full set
+// collector -> collector.hasFullSet(),
+// //create a new results collector, based on requested size
+// () -> new ResultsCollector( selectSize ),
+// //collect each candidate into our set of Candidate Results
+// ( collector, group ) -> {
+//
+// final List<CandidateResult> list = group.toList().toBlocking().last();
+//
+// collector.addCandidates( list );
+// } ) );
+//
+
+ //now reduce these results again via a cassandra collector
+
+
+
+ @Test
+ public void testLoops(){
+
+ final Observable<Integer> observable = Observable.create( new Observable.OnSubscribe<Integer>() {
+
+ int value = 0;
+
+
+ @Override
+ public void call( final Subscriber<? super Integer> subscriber ) {
+ subscriber.onNext( ++value );
+ }
+ } );
+
+
+
+
+ final CounterCollector collector = new CounterCollector();
+
+ final Action2<CounterCollector, Integer> collectorFunction = (col, value) -> {col.set( value ); };
+
+
- final ApplicationEntityIndex index = null;
+ final Func0<Boolean> complete = () -> collector.isFull();
- final SearchTypes searchType = SearchTypes.fromTypes( "test" );
+ final Observable<CounterCollector> collectorObservable = Statement.doWhile( observable, complete ).collect( () -> collector, collectorFunction );
- final String query = "select * ";
- final Observable<CandidateResults> observable = Observable.just( rootId ).map( createSearchScope( "type" ) ).map(getCandidates(index, searchType, query));
+ final int value = collectorObservable.toBlocking().last().lastValue;
- observable.doOnNext( a -> System.out.println( a) ).toBlocking().last();
+ assertEquals(100, value);
+
+
+
+
+
+ }
+
+
+ private static class CounterCollector{
+ private int lastValue;
+
+ public void set(final int newValue){
+ lastValue = newValue;
+ }
+
+ public boolean isFull(){
+ return lastValue >= 100;
+ }
}
- private static final class ResultsCollector{
+ private static final class ResultsCollector {
+
+ private Map<Id, IndexResult<CandidateResult>> toKeep = new HashMap<>();
+ private Set<CandidateResult> toRemove = new HashSet<>();
+
+ private final int resultSetSize;
+
+
+ private int currentIndex;
+
+
+ private ResultsCollector( final int resultSetSize ) {this.resultSetSize = resultSetSize;}
+
/**
* Add the candidates to our collection
- * @param results
*/
- public void addCandidates(final CandidateResults results ){
+ public void addCandidates( final List<CandidateResult> results ) {
+
+ Collections.sort( results, CandidateVersionComparator::compare );
+
+ final CandidateResult maxCandidate = results.get( 0 );
+
+ //add everything we don't use to our toRemoveSet
+ for ( int i = 1; i < results.size(); i++ ) {
+ toRemove.add( results.get( i ) );
+ }
+
+ //we have this id already, remove it and pick the max
+ final Id maxId = maxCandidate.getId();
+
+ //see if it exists in our set to keep
+ final IndexResult<CandidateResult> existingCandidate = toKeep.get( maxId );
+
+ if ( existingCandidate != null ) {
+
+ final CandidateResult existing = existingCandidate.value;
+
+ //our new value is greater than our existing, replace it
+ if ( CandidateVersionComparator.compare( maxCandidate, existing ) > 0 ) {
+ //add it to the keep
+ toKeep.put( maxId, new IndexResult( currentIndex, maxCandidate ) );
- //TODO, collect the results, removing groups
- Observable.from( results ).groupBy( candidate -> candidate.getId() ).collect( )
+ //remove the old value
+ toRemove.add( existingCandidate.value );
+ }
+
+ //what's in the map is already the max, add our candidate to the cleanup list
+ else {
+ toRemove.add( maxCandidate );
+ }
+ }
+
+ //add it to our list of items to keep
+ else {
+ toKeep.put( maxId, new IndexResult( currentIndex, maxCandidate ) );
+ }
+
+
+ //increment our index for the next invocation
+ currentIndex++;
+ }
+
+
+ /**
+ * We have a full set to evaluate for candidates
+ */
+ public boolean hasFullSet() {
+ return resultSetSize >= toKeep.size();
}
+
+ /**
+ * Get all the canidates we've collected
+ */
+ public Collection<IndexResult<CandidateResult>> getCandidates() {
+ return toKeep.values();
+ }
+
+ public Collection<CandidateResult> getStaleCandidates(){ return toRemove;}
}
+
+ /**
+ * Compares 2 candidates by version. The max version is considered greater
+ */
+ private static final class CandidateVersionComparator {
+
+ public static int compare( final CandidateResult o1, final CandidateResult o2 ) {
+ return UUIDComparator.staticCompare( o1.getVersion(), o2.getVersion() );
+ }
+ }
+
+
+ /**
+ * A data message with an index and a result. Can be used in aggregations
+ * @param <T>
+ */
+ private static final class IndexResult<T>{
+ private final int index;
+
+ private final T value;
+
+
+ private IndexResult( final int index, final T value ) {
+ this.index = index;
+ this.value = value;
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/61aa9794/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
index ce12429..b6ebcaa 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/rx/impl/CollectUntilTest.java
@@ -33,10 +33,10 @@ public class CollectUntilTest {
public void testCollectUntil() {
final CollectUntil<Integer, CountCollector> collectUntil =
- new CollectUntil<>(
+ new CollectUntil<>( collector -> collector.isFull() ,
() -> new CountCollector(),
- ( collector, value ) -> collector.mark(),
- collector -> collector.isFull() );
+ ( collector, value ) -> collector.mark()
+ );
final CountCollector collector = Observable.range( 0, 200 ).compose( collectUntil ).toBlocking().last();