You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/10 00:04:21 UTC
[01/16] Changes to repair to only bring us up to the current version
only. This is what we need in practice.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o 8b0355fee -> a336b6b40
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java
index 3643bd1..ab0e2f4 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java
@@ -17,6 +17,7 @@ import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.stage.AbstractIdStageTest;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.mvcc.stage.TestEntityGenerator;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.service.UUIDService;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -46,6 +47,8 @@ public class LoadTest extends AbstractIdStageTest {
final CollectionScope collection = mock(CollectionScope.class);
final UUIDService uuidService = mock(UUIDService.class);
final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
+ final SerializationFig serializationFig = mock(SerializationFig.class);
+ when(serializationFig.getBufferSize()).thenReturn( 10 );
final UUID loadVersion = UUIDGenerator.newTimeUUID();
@@ -69,7 +72,7 @@ public class LoadTest extends AbstractIdStageTest {
- Load load = new Load( uuidService, serializationStrategy );
+ Load load = new Load( uuidService, serializationStrategy, serializationFig );
Entity loaded = load.call( entityIoEvent );
@@ -87,6 +90,9 @@ public class LoadTest extends AbstractIdStageTest {
final CollectionScope collection = mock(CollectionScope.class);
final UUIDService uuidService = mock(UUIDService.class);
final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
+ final SerializationFig serializationFig = mock(SerializationFig.class);
+ when(serializationFig.getBufferSize()).thenReturn( 10 );
+
final UUID loadVersion = UUIDGenerator.newTimeUUID();
@@ -128,7 +134,7 @@ public class LoadTest extends AbstractIdStageTest {
when(serializationStrategy.write( same(collection), writtenEntityArg.capture() )).thenReturn( batch );
- Load load = new Load( uuidService, serializationStrategy );
+ Load load = new Load( uuidService, serializationStrategy, serializationFig );
Entity loaded = load.call( entityIoEvent );
assertNotNull( loaded.getField( "derp" ) );
@@ -162,6 +168,9 @@ public class LoadTest extends AbstractIdStageTest {
final CollectionScope collection = mock(CollectionScope.class);
final UUIDService uuidService = mock(UUIDService.class);
final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
+ final SerializationFig serializationFig = mock(SerializationFig.class);
+ when(serializationFig.getBufferSize()).thenReturn( 10 );
+
final ChangeLogGenerator changeLogGenerator = new ChangeLogGeneratorImpl();
@@ -206,7 +215,7 @@ public class LoadTest extends AbstractIdStageTest {
when(serializationStrategy.write( same(collection), writtenEntityArg.capture() )).thenReturn( batch );
- Load load = new Load( uuidService, serializationStrategy );
+ Load load = new Load( uuidService, serializationStrategy, serializationFig );
Entity loaded = load.call( entityIoEvent );
assertNotNull( loaded.getField( "derple" ) );
@@ -236,6 +245,9 @@ public class LoadTest extends AbstractIdStageTest {
final CollectionScope collection = mock(CollectionScope.class);
final UUIDService uuidService = mock(UUIDService.class);
final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
+ final SerializationFig serializationFig = mock(SerializationFig.class);
+ when(serializationFig.getBufferSize()).thenReturn( 10 );
+
final UUID loadVersion = UUIDGenerator.newTimeUUID();
@@ -284,7 +296,7 @@ public class LoadTest extends AbstractIdStageTest {
when(serializationStrategy.write( same(collection), writtenEntityArg.capture() )).thenReturn( batch );
- Load load = new Load( uuidService, serializationStrategy );
+ Load load = new Load( uuidService, serializationStrategy, serializationFig );
Entity loaded = load.call( entityIoEvent );
assertNotNull( loaded.getField( "derp" ) );
@@ -317,6 +329,9 @@ public class LoadTest extends AbstractIdStageTest {
final CollectionScope collection = mock(CollectionScope.class);
final UUIDService uuidService = mock(UUIDService.class);
final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
+ final SerializationFig serializationFig = mock(SerializationFig.class);
+ when(serializationFig.getBufferSize()).thenReturn( 10 );
+
final UUID loadVersion = UUIDGenerator.newTimeUUID();
@@ -334,7 +349,7 @@ public class LoadTest extends AbstractIdStageTest {
//mock up returning a list of MvccEntities
when(serializationStrategy.load( collection, entityId, loadVersion, 1 )).thenReturn( results);
- Load load = new Load( uuidService, serializationStrategy );
+ Load load = new Load( uuidService, serializationStrategy, serializationFig );
Entity loaded = load.call( entityIoEvent );
assertNull( "No entity was loaded", loaded );
@@ -347,6 +362,9 @@ public class LoadTest extends AbstractIdStageTest {
@Override
protected void validateStage( final CollectionIoEvent<Id> event ) {
final UUIDService uuidService = mock(UUIDService.class);
+ final SerializationFig serializationFig = mock(SerializationFig.class);
+ when(serializationFig.getBufferSize()).thenReturn( 10 );
+
/**
* Mock up wrong UUID type
@@ -356,6 +374,6 @@ public class LoadTest extends AbstractIdStageTest {
final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
- new Load(uuidService, serializationStrategy).call( event );
+ new Load(uuidService, serializationStrategy, serializationFig).call( event );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
new file mode 100644
index 0000000..f4a75d4
--- /dev/null
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
@@ -0,0 +1,136 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.collection.serialization;
+
+
+import java.util.Arrays;
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.impl.EntityRepairImpl;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import static org.junit.Assert.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+/**
+ * Tests the entity repair task
+ */
+public class EntityRepairImplTest {
+
+
+ /**
+ * Tests changing from a full version to 2 updates, ensures we have a proper output
+ */
+ @Test
+ public void testSimpleRolling() {
+
+ final SerializationFig serializationFig = mock( SerializationFig.class );
+
+ when( serializationFig.getBufferSize() ).thenReturn( 10 );
+
+
+ final Id simpleId = new SimpleId( "entity" );
+
+ final Entity v1Entity = new Entity( simpleId );
+
+
+ v1Entity.setField( new StringField( "field1", "value1" ) );
+ v1Entity.setField( new StringField( "field2", "value2" ) );
+
+
+ final MvccEntityImpl v1 =
+ new MvccEntityImpl( simpleId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.COMPLETE, v1Entity );
+
+
+ final Entity v2Entity = new Entity( simpleId );
+ v2Entity.setField( new StringField( "field1", "value1.1" ) );
+
+ final MvccEntityImpl v2 =
+ new MvccEntityImpl( simpleId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, v2Entity );
+
+
+ final Entity v3Entity = new Entity( simpleId );
+ v3Entity.setField( new StringField( "field2", "value1.2" ) );
+
+ final MvccEntityImpl v3 =
+ new MvccEntityImpl( simpleId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, v3Entity );
+
+
+ final MvccEntitySerializationStrategy mvccEntitySerializationStrategy =
+ mock( MvccEntitySerializationStrategy.class );
+
+
+ final Id applicationId = new SimpleId( "application" );
+
+ final CollectionScope scope = new CollectionScopeImpl( applicationId, applicationId, "users" );
+
+ //mock up returning
+ when( mvccEntitySerializationStrategy
+ .load( scope, simpleId, v3.getVersion(), serializationFig.getBufferSize() ) )
+ .thenReturn( Arrays.<MvccEntity>asList( v3, v2, v1 ).iterator() );
+
+
+ EntityRepairImpl entityRepair = new EntityRepairImpl( mvccEntitySerializationStrategy, serializationFig );
+
+ final MvccEntity returned = entityRepair.maybeRepair( scope, v3 );
+
+ final UUID version = returned.getVersion();
+
+ assertEquals( "Versions should match", v3.getVersion(), version );
+
+
+
+ final Id entityId = returned.getId();
+
+ assertEquals( "Entity Id's match", simpleId, entityId );
+
+
+
+ final Entity finalVersion = returned.getEntity().get();
+
+ final Object expectedField1Value = v2.getEntity().get().getField( "field1" ).getValue();
+
+ final Object returnedField1Value = finalVersion.getField( "field1" ).getValue();
+
+ assertEquals( "Same field value", expectedField1Value, returnedField1Value );
+
+
+
+ final Object expectedField2Value = v3.getEntity().get().getField( "field2" ).getValue();
+
+ final Object returnedField2Value = finalVersion.getField( "field2" ).getValue();
+
+ assertEquals( "Same field value", expectedField2Value, returnedField2Value );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index 6780115..ba5ebce 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -649,7 +649,7 @@ public class MvccEntitySerializationStrategyImplTest {
public void loadParamEntityId() throws ConnectionException {
serializationStrategy
- .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), null, UUIDGenerator.newTimeUUID() );
+ .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), (Id)null, UUIDGenerator.newTimeUUID() );
}
[03/16] git commit: Refactored read stage
Posted by to...@apache.org.
Refactored read stage
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f1a88aa9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f1a88aa9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f1a88aa9
Branch: refs/heads/two-dot-o
Commit: f1a88aa908211e0bb2c137f96c2fe0028ff202ac
Parents: f7d406e
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 11:27:23 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 11:28:56 2014 -0600
----------------------------------------------------------------------
.../persistence/collection/EntitySet.java | 3 +-
.../impl/EntityCollectionManagerImpl.java | 45 ++-
.../mvcc/MvccEntitySerializationStrategy.java | 12 -
.../collection/mvcc/stage/load/Load.java | 111 ------
.../serialization/SerializationFig.java | 19 +
.../serialization/impl/EntityRepairImpl.java | 5 +-
.../serialization/impl/EntitySetImpl.java | 10 +-
.../MvccEntitySerializationStrategyImpl.java | 32 +-
.../collection/EntityCollectionManagerIT.java | 240 +++++++++---
.../collection/mvcc/stage/load/LoadTest.java | 379 -------------------
.../serialization/EntityRepairImplTest.java | 25 +-
...MvccEntitySerializationStrategyImplTest.java | 40 +-
12 files changed, 290 insertions(+), 631 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
index 32c168f..7803a04 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence.collection;/*
*/
+import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -32,7 +33,7 @@ public interface EntitySet {
* @param entityId
* @return
*/
- public Entity getEntity(Id entityId);
+ public MvccEntity getEntity(Id entityId);
/**
* Get the number of entities in this set
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 4a72285..39df9cb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.collection.impl;
import java.util.Collection;
+import java.util.Collections;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -30,12 +31,12 @@ import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
-import org.apache.usergrid.persistence.collection.mvcc.stage.load.Load;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
@@ -46,6 +47,7 @@ import org.apache.usergrid.persistence.core.task.TaskExecutor;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
@@ -79,9 +81,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final WriteCommit writeCommit;
private final RollbackAction rollback;
- //load stages
- private final Load load;
-
//delete stages
private final MarkStart markStart;
@@ -89,16 +88,20 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final TaskExecutor taskExecutor;
+ private final MvccEntitySerializationStrategy entitySerializationStrategy;
+
@Inject
public EntityCollectionManagerImpl( final UUIDService uuidService, @Write final WriteStart writeStart,
@WriteUpdate final WriteStart writeUpdate,
final WriteUniqueVerify writeVerifyUnique,
final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback, final Load load,
+ final WriteCommit writeCommit, final RollbackAction rollback,
final MarkStart markStart, final MarkCommit markCommit,
- @CollectionTaskExecutor
- final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope) {
+ final MvccEntitySerializationStrategy entitySerializationStrategy,
+ @CollectionTaskExecutor final TaskExecutor taskExecutor,
+ @Assisted final CollectionScope collectionScope
+ ) {
+
Preconditions.checkNotNull( uuidService, "uuidService must be defined" );
@@ -111,13 +114,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.writeCommit = writeCommit;
this.rollback = rollback;
- this.load = load;
this.markStart = markStart;
this.markCommit = markCommit;
this.uuidService = uuidService;
this.collectionScope = collectionScope;
this.taskExecutor = taskExecutor;
+ this.entitySerializationStrategy = entitySerializationStrategy;
}
@@ -183,14 +186,30 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
- return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) )
- .map( load );
+ return load( Collections.singleton(entityId) ).map( new Func1<EntitySet, Entity>() {
+ @Override
+ public Entity call( final EntitySet entitySet ) {
+ final MvccEntity entity = entitySet.getEntity( entityId );
+
+ if(entity == null){
+ return null;
+ }
+
+ return entity.getEntity().orNull();
+ }
+ } );
}
@Override
public Observable<EntitySet> load( final Collection<Id> entityIds ) {
- return null;
+
+ Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
+
+ final EntitySet
+ results = entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
+
+ return Observable.just( results );
}
@@ -222,7 +241,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
//we an update, signal the fix
//TODO T.N Change this to fire a task
- Observable.from( new CollectionIoEvent<Id>(collectionScope, entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
+// Observable.from( new CollectionIoEvent<Id>(collectionScope, entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
index 7947029..0467c95 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
@@ -47,18 +47,6 @@ public interface MvccEntitySerializationStrategy {
public MutationBatch write( CollectionScope context, MvccEntity entity );
- /**
- * Load and return the entity with the given id and a version that is <= the version provided
- *
- * @param context The context to persist the entity into
- * @param entityId The entity id to load
- * @param version The version to load. This will return the version <= the given version
- *
- * @return The deserialized version of the entity. Null if no version == to version exists. If the entity version
- * has been cleared, the MvccEntity will be returned, but the optional entity will not be set
- */
- public MvccEntity load( CollectionScope context, Id entityId, UUID version );
-
/**
* Load the entities into the entitySet from the specified Ids. Loads versions <= the maxVersion
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java
deleted file mode 100644
index f9c66cc..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.collection.mvcc.stage.load;
-
-
-import java.util.Iterator;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
-import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.serialization.impl.EntityRepairImpl;
-import org.apache.usergrid.persistence.collection.service.UUIDService;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Preconditions;
-import com.google.inject.Inject;
-import com.google.inject.Singleton;
-
-import rx.functions.Func1;
-
-
-/**
- * This stage is a load stage to load a single entity
- */
-@Singleton
-public class Load implements Func1<CollectionIoEvent<Id>, Entity> {
-
-
- private static final Logger LOG = LoggerFactory.getLogger( Load.class );
-
- private final UUIDService uuidService;
- private final MvccEntitySerializationStrategy entitySerializationStrategy;
- private final EntityRepair entityRepair;
-
-
- @Inject
- public Load( final UUIDService uuidService, final MvccEntitySerializationStrategy entitySerializationStrategy, final
- SerializationFig serializationFig ) {
- Preconditions.checkNotNull( entitySerializationStrategy, "entitySerializationStrategy is required" );
- Preconditions.checkNotNull( uuidService, "uuidService is required" );
-
-
- this.uuidService = uuidService;
- this.entitySerializationStrategy = entitySerializationStrategy;
- entityRepair = new EntityRepairImpl( entitySerializationStrategy, serializationFig );
-
- }
-
- //TODO: do reads partial merges in batches. maybe 5 or 10 at a time.
- /**
- * for example
- so if like v1 is a full
- and you have v1 -> v20, where v2->20 is all partial
- you merge up to 10, then flush
- then process 10->20, then flush
- */
- @Override
- public Entity call( final CollectionIoEvent<Id> idIoEvent ) {
- final Id entityId = idIoEvent.getEvent();
-
- ValidationUtils.verifyIdentity( entityId );
-
-
- final CollectionScope collectionScope = idIoEvent.getEntityCollection();
-
- //generate a version that represents now
- final UUID versionMax = uuidService.newTimeUUID();
-
- Iterator<MvccEntity> results = entitySerializationStrategy.load(
- collectionScope, entityId, versionMax, 1 );
-
- if(!results.hasNext()){
- return null;
- }
-
- final MvccEntity returned = results.next();
-
- final MvccEntity repairedEntity = entityRepair.maybeRepair( collectionScope, returned );
-
- if(repairedEntity == null)
- return null;
-
- return repairedEntity.getEntity().orNull();
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
index 7e69a19..15bae5c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/SerializationFig.java
@@ -56,4 +56,23 @@ public interface SerializationFig extends GuicyFig {
@Key( "collection.task.pool.queuesize" )
@Default( "20" )
int getTaskPoolQueueSize();
+
+ /**
+ * The maximum amount of entites we can load at any one time
+ * @return
+ */
+ @Key( "collection.max.load.size" )
+ @Default( "1000" )
+ int getMaxLoadSize();
+
+
+
+ /**
+ * The maximum number of bytes a serialized entity can be. Any thing beyond this is rejected
+ * @return
+ */
+ @Key( "collection.max.entry.size" )
+ @Default( "512" )
+ int getMaxEntrySize();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
index d26cd3f..62f4915 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
@@ -36,6 +36,7 @@ import org.apache.usergrid.persistence.model.field.Field;
import com.google.inject.Inject;
import com.google.inject.Singleton;
+import com.netflix.astyanax.MutationBatch;
@Singleton
@@ -102,7 +103,9 @@ public class EntityRepairImpl implements EntityRepair {
final MvccEntity mergedEntity = entityRepair( changeLog, targetEntity );
try {
- mvccEntitySerializationStrategy.write( collectionScope, mergedEntity ).execute();
+ final MutationBatch batch = mvccEntitySerializationStrategy.write( collectionScope, mergedEntity );
+
+ batch.execute();
}
catch ( Exception e ) {
throw new RuntimeException( "Couldn't rewrite repaired entity", e );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
index 89ec69e..9e03754 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
@@ -45,14 +45,8 @@ public class EntitySetImpl implements EntitySet {
@Override
- public Entity getEntity( final Id entityId ) {
- final MvccEntity entity = entities.get( entityId );
-
- if ( entity == null ) {
- return null;
- }
-
- return entity.getEntity().orNull();
+ public MvccEntity getEntity( final Id entityId ) {
+ return entities.get( entityId );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index b36ee0e..5449e46 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -142,33 +142,6 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
}
- @Override
- public MvccEntity load( final CollectionScope collectionScope, final Id entityId, final UUID version ) {
- Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
- Preconditions.checkNotNull( entityId, "entity id is required" );
- Preconditions.checkNotNull( version, "version is required" );
-
-
- Column<UUID> column;
-
- try {
- column = keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( ScopedRowKey.fromKey( collectionScope, entityId ) )
- .getColumn( version ).execute().getResult();
- }
-
- catch ( NotFoundException e ) {
- //swallow, there's just no column
- return null;
- }
- catch ( ConnectionException e ) {
- throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra",
- e );
- }
-
-
- return new MvccColumnParser( entityId ).parseColumn( column );
- }
-
@Override
public EntitySet load( final CollectionScope collectionScope, final Collection<Id> entityIds,
@@ -181,6 +154,11 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
Preconditions.checkNotNull( maxVersion, "version is required" );
+ //didnt put the max in the error message, I don't want to take the string construction hit every time
+ Preconditions.checkArgument( entityIds.size() <= serializationFig.getMaxLoadSize(), "requested size cannot be over configured maximum");
+
+
+
final List<ScopedRowKey<CollectionScope, Id>> rowKeys = new ArrayList<>( entityIds.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 9e8bbe3..823cb87 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -18,6 +18,8 @@
package org.apache.usergrid.persistence.collection;
+import java.util.ArrayList;
+import java.util.List;
import java.util.UUID;
import org.jukito.UseModules;
@@ -29,20 +31,27 @@ import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyExc
import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.BooleanField;
import org.apache.usergrid.persistence.model.field.IntegerField;
import org.apache.usergrid.persistence.model.field.StringField;
import com.fasterxml.uuid.UUIDComparator;
-import com.fasterxml.uuid.impl.UUIDUtil;
import com.google.inject.Inject;
import rx.Observable;
-
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/** @author tnine */
@@ -57,12 +66,15 @@ public class EntityCollectionManagerIT {
@Rule
public MigrationManagerRule migrationManagerRule;
+ @Inject
+ private SerializationFig serializationFig;
+
@Test
public void write() {
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
Entity newEntity = new Entity( new SimpleId( "test" ) );
@@ -78,17 +90,18 @@ public class EntityCollectionManagerIT {
assertNotNull( "Version exists", returned.getVersion() );
}
+
@Test
public void writeWithUniqueValues() {
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
EntityCollectionManager manager = factory.createCollectionManager( context );
{
Entity newEntity = new Entity( new SimpleId( "test" ) );
- newEntity.setField( new IntegerField("count", 5, true) );
+ newEntity.setField( new IntegerField( "count", 5, true ) );
Observable<Entity> observable = manager.write( newEntity );
Entity returned = observable.toBlocking().lastOrDefault( null );
@@ -97,23 +110,24 @@ public class EntityCollectionManagerIT {
{
try {
Entity newEntity = new Entity( new SimpleId( "test" ) );
- newEntity.setField( new IntegerField("count", 5, true) );
+ newEntity.setField( new IntegerField( "count", 5, true ) );
manager.write( newEntity ).toBlocking().last();
- fail("Write should have thrown an exception");
-
- } catch ( Exception ex ) {
- WriteUniqueVerifyException e = (WriteUniqueVerifyException)ex;
+ fail( "Write should have thrown an exception" );
+ }
+ catch ( Exception ex ) {
+ WriteUniqueVerifyException e = ( WriteUniqueVerifyException ) ex;
assertEquals( 1, e.getVioliations().size() );
}
}
}
+
@Test
public void writeAndLoad() {
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
Entity newEntity = new Entity( new SimpleId( "test" ) );
@@ -139,8 +153,8 @@ public class EntityCollectionManagerIT {
@Test
public void writeLoadDelete() {
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
Entity newEntity = new Entity( new SimpleId( "test" ) );
EntityCollectionManager manager = factory.createCollectionManager( context );
@@ -174,8 +188,8 @@ public class EntityCollectionManagerIT {
@Test
public void writeLoadUpdateLoad() {
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
Entity newEntity = new Entity( new SimpleId( "test" ) );
newEntity.setField( new IntegerField( "counter", 1 ) );
@@ -221,8 +235,8 @@ public class EntityCollectionManagerIT {
public void writeAndLoadScopeClosure() {
- CollectionScope collectionScope1 = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test1" ), "test1" );
+ CollectionScope collectionScope1 =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test1" ), "test1" );
Entity newEntity = new Entity( new SimpleId( "test" ) );
@@ -245,30 +259,32 @@ public class EntityCollectionManagerIT {
//now make sure we can't load it from another scope, using the same org
- CollectionScope collectionScope2 = new CollectionScopeImpl(
- collectionScope1.getApplication(), new SimpleId("test2"), collectionScope1.getName());
+ CollectionScope collectionScope2 =
+ new CollectionScopeImpl( collectionScope1.getApplication(), new SimpleId( "test2" ),
+ collectionScope1.getName() );
EntityCollectionManager manager2 = factory.createCollectionManager( collectionScope2 );
- Entity loaded = manager2.load( createReturned.getId() )
- .toBlocking().lastOrDefault( null );
+ Entity loaded = manager2.load( createReturned.getId() ).toBlocking().lastOrDefault( null );
assertNull( "CollectionScope works correctly", loaded );
//now try to load it from another org, with the same scope
- CollectionScope collectionScope3 = new CollectionScopeImpl(
- new SimpleId("organization2"), collectionScope1.getOwner(), collectionScope1.getName());
+ CollectionScope collectionScope3 =
+ new CollectionScopeImpl( new SimpleId( "organization2" ), collectionScope1.getOwner(),
+ collectionScope1.getName() );
assertNotNull( collectionScope3 );
}
+
@Test
public void partialUpdate() {
- StringField testField1 = new StringField("testField","value");
+ StringField testField1 = new StringField( "testField", "value" );
StringField addedField = new StringField( "testFud", "NEWPARTIALUPDATEZOMG" );
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "testUpdate" ), "testUpdate" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "testUpdate" ), "testUpdate" );
Entity oldEntity = new Entity( new SimpleId( "testUpdate" ) );
oldEntity.setField( new StringField( "testField", "value" ) );
@@ -283,12 +299,12 @@ public class EntityCollectionManagerIT {
final UUID writeVersion = returned.getVersion();
- assertNotNull("Write version was set", writeVersion);
+ assertNotNull( "Write version was set", writeVersion );
/**
* Modify the oldEntity
*/
- oldEntity.getFields().remove( testField1 );
+ oldEntity.getFields().remove( testField1 );
oldEntity.setField( addedField );
observable = manager.update( oldEntity );
@@ -300,7 +316,7 @@ public class EntityCollectionManagerIT {
final UUID updatedVersion = updateReturned.getVersion();
- assertNotNull("Updated version returned", updatedVersion);
+ assertNotNull( "Updated version returned", updatedVersion );
assertTrue( "Updated version higher", UUIDComparator.staticCompare( updatedVersion, writeVersion ) > 0 );
@@ -309,21 +325,20 @@ public class EntityCollectionManagerIT {
final UUID returnedVersion = newEntity.getVersion();
- assertEquals("Loaded version matches updated version", updatedVersion, returnedVersion);
+ assertEquals( "Loaded version matches updated version", updatedVersion, returnedVersion );
assertNotNull( "Returned has a uuid", returned.getId() );
- assertEquals( addedField, newEntity.getField( "testFud" ));
-
-
+ assertEquals( addedField, newEntity.getField( "testFud" ) );
}
+
@Test
public void partialUpdateDelete() {
- StringField testField = new StringField("testField","value");
+ StringField testField = new StringField( "testField", "value" );
StringField addedField = new StringField( "testFud", "NEWPARTIALUPDATEZOMG" );
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "testUpdate" ), "testUpdate" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "testUpdate" ), "testUpdate" );
Entity oldEntity = new Entity( new SimpleId( "testUpdate" ) );
oldEntity.setField( new StringField( "testField", "value" ) );
@@ -336,13 +351,13 @@ public class EntityCollectionManagerIT {
assertNotNull( "Returned has a uuid", returned.getId() );
- oldEntity.getFields().remove( testField );
+ oldEntity.getFields().remove( testField );
oldEntity.setField( addedField );
//Entity is deleted then updated right afterwards.
manager.delete( oldEntity.getId() );
- observable = manager.update( oldEntity);
+ observable = manager.update( oldEntity );
returned = observable.toBlocking().lastOrDefault( null );
@@ -353,9 +368,7 @@ public class EntityCollectionManagerIT {
Entity newEntity = newEntityObs.toBlocking().last();
assertNotNull( "Returned has a uuid", returned.getId() );
- assertEquals( addedField, newEntity.getField( addedField.getName()) );
-
-
+ assertEquals( addedField, newEntity.getField( addedField.getName() ) );
}
@@ -366,10 +379,10 @@ public class EntityCollectionManagerIT {
Entity origEntity = new Entity( new SimpleId( "testUpdate" ) );
origEntity.setField( new StringField( "testField", "value" ) );
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "testUpdate" ), "testUpdate" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "testUpdate" ), "testUpdate" );
EntityCollectionManager manager = factory.createCollectionManager( context );
- Entity returned = manager.write(origEntity).toBlocking().lastOrDefault( null );
+ Entity returned = manager.write( origEntity ).toBlocking().lastOrDefault( null );
// note its version
UUID oldVersion = returned.getVersion();
@@ -379,17 +392,140 @@ public class EntityCollectionManagerIT {
// partial update entity but we don't have version number
Entity updateEntity = new Entity( origEntity.getId() );
- updateEntity.setField( new StringField("addedField", "other value" ) );
- manager.update(origEntity).toBlocking().lastOrDefault(null);
+ updateEntity.setField( new StringField( "addedField", "other value" ) );
+ manager.update( origEntity ).toBlocking().lastOrDefault( null );
// get entity now, it must have a new version
- returned = manager.load(origEntity.getId() ).toBlocking().lastOrDefault( null );
+ returned = manager.load( origEntity.getId() ).toBlocking().lastOrDefault( null );
UUID newVersion = returned.getVersion();
assertNotNull( "A new version must be assigned", newVersion );
// new Version should be > old version
- assertTrue( UUIDComparator.staticCompare( newVersion, oldVersion ) > 0);
+ assertTrue( UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
+ }
+
+
+ @Test
+ public void writeMultiget() {
+
+ final CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final int multigetSize = serializationFig.getMaxLoadSize();
+
+ final List<Entity> writtenEntities = new ArrayList<>( multigetSize );
+ final List<Id> entityIds = new ArrayList<>( multigetSize );
+
+ for ( int i = 0; i < multigetSize; i++ ) {
+ final Entity entity = new Entity( new SimpleId( "test" ) );
+
+ final Entity written = manager.write( entity ).toBlocking().last();
+
+ writtenEntities.add( written );
+ entityIds.add( written.getId() );
+ }
+
+
+ final EntitySet entitySet = manager.load( entityIds ).toBlocking().lastOrDefault( null );
+
+ assertNotNull( entitySet );
+
+ assertEquals(multigetSize, entitySet.size());
+ assertFalse(entitySet.isEmpty());
+
+ /**
+ * Validate every element exists
+ */
+ for(int i = 0; i < multigetSize; i ++){
+ final Entity expected = writtenEntities.get( i );
+
+ final MvccEntity returned = entitySet.getEntity( expected.getId() );
+
+ assertEquals("Same entity returned", expected, returned.getEntity().get());
+ }
+
+
}
+
+ /**
+ * Perform a multiget where every entity will need repaired on load
+ */
+ @Test
+ public void writeMultigetRepair() {
+
+ final CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final int multigetSize = serializationFig.getMaxLoadSize();
+
+ final List<Entity> writtenEntities = new ArrayList<>( multigetSize );
+ final List<Id> entityIds = new ArrayList<>( multigetSize );
+
+ for ( int i = 0; i < multigetSize; i++ ) {
+ final Entity entity = new Entity( new SimpleId( "test" ) );
+
+ final Entity written = manager.write( entity ).toBlocking().last();
+
+ written.setField( new BooleanField( "updated", true ) );
+
+ final Entity updated = manager.update( written ).toBlocking().last();
+
+ writtenEntities.add( updated );
+ entityIds.add( updated.getId() );
+ }
+
+
+ final EntitySet entitySet = manager.load( entityIds ).toBlocking().lastOrDefault( null );
+
+ assertNotNull( entitySet );
+
+ assertEquals(multigetSize, entitySet.size());
+ assertFalse(entitySet.isEmpty());
+
+ /**
+ * Validate every element exists
+ */
+ for(int i = 0; i < multigetSize; i ++){
+ final Entity expected = writtenEntities.get( i );
+
+ final MvccEntity returned = entitySet.getEntity( expected.getId() );
+
+ assertEquals("Same entity returned", expected, returned.getEntity().get());
+
+ assertTrue( ( Boolean ) returned.getEntity().get().getField( "updated" ).getValue() );
+ }
+
+
+ }
+
+
+
+ @Test(expected = IllegalArgumentException.class)
+ public void readTooLarge() {
+
+ final CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final int multigetSize = serializationFig.getMaxLoadSize() +1;
+
+
+ final List<Id> entityIds = new ArrayList<>( multigetSize );
+
+ for ( int i = 0; i < multigetSize; i++ ) {
+
+ entityIds.add( new SimpleId( "simple" ) );
+ }
+
+
+ //should throw an exception
+ manager.load( entityIds ).toBlocking().lastOrDefault( null );
+
+
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java
deleted file mode 100644
index ab0e2f4..0000000
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/LoadTest.java
+++ /dev/null
@@ -1,379 +0,0 @@
-package org.apache.usergrid.persistence.collection.mvcc.stage.load;
-
-
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
-import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.stage.AbstractIdStageTest;
-import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
-import org.apache.usergrid.persistence.collection.mvcc.stage.TestEntityGenerator;
-import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
-import org.apache.usergrid.persistence.collection.service.UUIDService;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-
-import com.google.common.collect.Lists;
-import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.mockito.Matchers.same;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-
-/** @author tnine */
-
-public class LoadTest extends AbstractIdStageTest {
-
-
- @Test
- public void testLoadWithData(){
- final CollectionScope collection = mock(CollectionScope.class);
- final UUIDService uuidService = mock(UUIDService.class);
- final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
- final SerializationFig serializationFig = mock(SerializationFig.class);
- when(serializationFig.getBufferSize()).thenReturn( 10 );
-
-
- final UUID loadVersion = UUIDGenerator.newTimeUUID();
-
- //mock up the time uuid
- when(uuidService.newTimeUUID()).thenReturn(loadVersion);
-
- final Id entityId = TestEntityGenerator.generateId();
-
- final CollectionIoEvent<Id> entityIoEvent = new CollectionIoEvent<Id>(collection, entityId );
-
-
- final Entity entity = TestEntityGenerator.generateEntity(entityId, loadVersion);
-
- final MvccEntity mvccEntity = TestEntityGenerator.fromEntityStatus( entity, MvccEntity.Status.COMPLETE );
-
- final Iterator<MvccEntity> results = Lists.newArrayList( mvccEntity ).iterator();
-
- //mock up returning a list of MvccEntities
- when(serializationStrategy.load( collection, entityId, loadVersion, 1 )).thenReturn( results);
-
-
-
- Load load = new Load( uuidService, serializationStrategy, serializationFig );
- Entity loaded = load.call( entityIoEvent );
-
-
- assertSame("Same entity was loaded", entity, loaded);
-
- }
-
-
- /**
- * Handles second trigger condition with partial updates.
- * A read on an entity , and we recognize that the entity we are reading is partial.
- */
- @Test
- public void testLoadWithPartialWrite() throws ConnectionException {
- final CollectionScope collection = mock(CollectionScope.class);
- final UUIDService uuidService = mock(UUIDService.class);
- final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
- final SerializationFig serializationFig = mock(SerializationFig.class);
- when(serializationFig.getBufferSize()).thenReturn( 10 );
-
-
-
- final UUID loadVersion = UUIDGenerator.newTimeUUID();
-
- //mock up the time uuid
- when(uuidService.newTimeUUID()).thenReturn(loadVersion);
-
- final Id entityId = TestEntityGenerator.generateId();
-
- final CollectionIoEvent<Id> entityIoEvent = new CollectionIoEvent<Id>(collection, entityId );
-
-
- final Entity entity = TestEntityGenerator.generateEntity(entityId, loadVersion);
- entity.setField( new StringField( "derp","noderp" ) );
-
- final MvccEntity completeMvccEntity = TestEntityGenerator.fromEntityStatus( entity, MvccEntity.Status.COMPLETE );
-
-
- final Entity entity2 = TestEntityGenerator.generateEntity( entityId, UUIDGenerator.newTimeUUID() );
- entity2.setField( new StringField( "derp","noderp" ) );
- entity2.setField( new StringField( "merple","nomerple" ) );
-
- final MvccEntity partialMvccEntity = TestEntityGenerator.fromEntityStatus( entity2, MvccEntity.Status.PARTIAL );
-
- final List<MvccEntity> results = Lists.newArrayList( partialMvccEntity );
- results.add( completeMvccEntity );
-
-
- //mock up returning a list of MvccEntities
- when( serializationStrategy.load( collection, entityId, loadVersion, 1 ) ).thenReturn( results.iterator());
-
- /**
- * Mock up returning the batch for repair
- */
-
- final MutationBatch batch = mock(MutationBatch.class);
-
- ArgumentCaptor<MvccEntity> writtenEntityArg = ArgumentCaptor.forClass( MvccEntity.class );
-
- when(serializationStrategy.write( same(collection), writtenEntityArg.capture() )).thenReturn( batch );
-
- Load load = new Load( uuidService, serializationStrategy, serializationFig );
- Entity loaded = load.call( entityIoEvent );
-
- assertNotNull( loaded.getField( "derp" ) );
- assertNull( loaded.getField( "derple" ) );
-
- /**
- * Verify that the repair executed
- */
- verify(batch).execute();
-
- /**
- * Verify the fields in the argument are correct
- */
- MvccEntity writtenEntity = writtenEntityArg.getValue();
-
- //assert all fields that should be updated
-
- assertSame( loaded, writtenEntity.getEntity().get() );
- assertNotNull( writtenEntity );
- assertNotNull( writtenEntity.getEntity().get().getField( "derp" ) );
- assertNull( writtenEntity.getEntity().get().getField( "derple" ) );
-
- }
-
- /**
- * Handles second trigger condition with partial updates.
- * A read on an entity , and we recognize that the entity we are reading is partial.
- */
- @Test
- public void testLoadWithPartialDelete() throws ConnectionException {
- final CollectionScope collection = mock(CollectionScope.class);
- final UUIDService uuidService = mock(UUIDService.class);
- final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
- final SerializationFig serializationFig = mock(SerializationFig.class);
- when(serializationFig.getBufferSize()).thenReturn( 10 );
-
- final ChangeLogGenerator changeLogGenerator = new ChangeLogGeneratorImpl();
-
-
-
- final UUID loadVersion = UUIDGenerator.newTimeUUID();
-
- //mock up the time uuid
- when(uuidService.newTimeUUID()).thenReturn(loadVersion);
-
- final Id entityId = TestEntityGenerator.generateId();
-
- final CollectionIoEvent<Id> entityIoEvent = new CollectionIoEvent<Id>(collection, entityId );
-
-
- final Entity entity = TestEntityGenerator.generateEntity(entityId, loadVersion);
- entity.setField( new StringField( "derp","noderp" ) );
- entity.setField( new StringField( "derple","somemerple" ) );
-
- final MvccEntity completeMvccEntity = TestEntityGenerator.fromEntityStatus( entity, MvccEntity.Status.COMPLETE );
-
-
- final Entity entity2 = TestEntityGenerator.generateEntity( entityId, UUIDGenerator.newTimeUUID() );
- entity2.setField( new StringField( "derple","somemerple" ) );
-
-
- final MvccEntity partialMvccEntity = TestEntityGenerator.fromEntityStatus( entity2, MvccEntity.Status.PARTIAL );
-
- final List<MvccEntity> results = Lists.newArrayList( partialMvccEntity );
- results.add( completeMvccEntity );
-
- //mock up returning a list of MvccEntities
- when( serializationStrategy.load( collection, entityId, loadVersion, 1 ) ).thenReturn( results.iterator());
-
-
- /**
- * Mock up returning the batch for repair
- */
-
- final MutationBatch batch = mock(MutationBatch.class);
-
- ArgumentCaptor<MvccEntity> writtenEntityArg = ArgumentCaptor.forClass( MvccEntity.class );
-
- when(serializationStrategy.write( same(collection), writtenEntityArg.capture() )).thenReturn( batch );
-
- Load load = new Load( uuidService, serializationStrategy, serializationFig );
- Entity loaded = load.call( entityIoEvent );
-
- assertNotNull( loaded.getField( "derple" ) );
-
-
- /**
- * Verify that the repair executed
- */
- verify(batch).execute();
-
- /**
- * Verify the fields in the argument are correct
- */
- MvccEntity writtenEntity = writtenEntityArg.getValue();
-
- //assert all fields that should be updated
-
- assertSame( loaded, writtenEntity.getEntity().get() );
- assertNotNull( writtenEntity );
- assertNotNull( writtenEntity.getEntity().get().getField( "derple" ) );
-
-
- }
-
- @Test
- public void testLoadWithPartialWriteDeleteThreeTimes() throws ConnectionException {
- final CollectionScope collection = mock(CollectionScope.class);
- final UUIDService uuidService = mock(UUIDService.class);
- final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
- final SerializationFig serializationFig = mock(SerializationFig.class);
- when(serializationFig.getBufferSize()).thenReturn( 10 );
-
-
- final UUID loadVersion = UUIDGenerator.newTimeUUID();
-
- //mock up the time uuid
- when(uuidService.newTimeUUID()).thenReturn(loadVersion);
-
- final Id entityId = TestEntityGenerator.generateId();
-
- final CollectionIoEvent<Id> entityIoEvent = new CollectionIoEvent<Id>(collection, entityId );
-
-
- final Entity entity = TestEntityGenerator.generateEntity(entityId, loadVersion);
- entity.setField( new StringField( "derp","noderp" ) );
- entity.setField( new StringField( "derple","somemerple" ) );
-
- final MvccEntity completeMvccEntity = TestEntityGenerator.fromEntityStatus( entity, MvccEntity.Status.COMPLETE );
-
-
- final Entity entity2 = TestEntityGenerator.generateEntity( entityId, UUIDGenerator.newTimeUUID() );
- entity2.setField( new StringField( "derple","somemerple" ) );
-
-
- final MvccEntity partialMvccEntity = TestEntityGenerator.fromEntityStatus( entity2, MvccEntity.Status.PARTIAL );
-
- final Entity entity3 = TestEntityGenerator.generateEntity( entityId, UUIDGenerator.newTimeUUID() );
- entity3.setField( new StringField( "derp","noderp" ) );
-
-
- final MvccEntity partialMvccEntity2 = TestEntityGenerator.fromEntityStatus( entity3, MvccEntity.Status.PARTIAL );
-
- final List<MvccEntity> results = Lists.newArrayList( partialMvccEntity2 );
- results.add( partialMvccEntity );
- results.add( completeMvccEntity );
-
- //mock up returning a list of MvccEntities
- when( serializationStrategy.load( collection, entityId, loadVersion, 1 ) ).thenReturn( results.iterator());
-
-
- /**
- * Mock up returning the batch for repair
- */
-
- final MutationBatch batch = mock(MutationBatch.class);
-
- ArgumentCaptor<MvccEntity> writtenEntityArg = ArgumentCaptor.forClass( MvccEntity.class );
-
- when(serializationStrategy.write( same(collection), writtenEntityArg.capture() )).thenReturn( batch );
-
- Load load = new Load( uuidService, serializationStrategy, serializationFig );
- Entity loaded = load.call( entityIoEvent );
-
- assertNotNull( loaded.getField( "derp" ) );
- assertNull( loaded.getField( "derple" ) );
-
- /**
- * Verify that the repair executed
- */
- verify(batch).execute();
-
- /**
- * Verify the fields in the argument are correct
- */
- MvccEntity writtenEntity = writtenEntityArg.getValue();
-
- //assert all fields that should be updated
-
- assertSame( loaded, writtenEntity.getEntity().get() );
- assertNotNull( writtenEntity );
- assertNotNull( writtenEntity.getEntity().get().getField( "derp" ) );
- assertNull( writtenEntity.getEntity().get().getField( "derple" ) );
-
-
-
-
- }
-
- @Test
- public void testLoadCleared(){
- final CollectionScope collection = mock(CollectionScope.class);
- final UUIDService uuidService = mock(UUIDService.class);
- final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
- final SerializationFig serializationFig = mock(SerializationFig.class);
- when(serializationFig.getBufferSize()).thenReturn( 10 );
-
-
-
- final UUID loadVersion = UUIDGenerator.newTimeUUID();
-
- //mock up the time uuid
- when(uuidService.newTimeUUID()).thenReturn(loadVersion);
-
- final Id entityId = TestEntityGenerator.generateId();
-
- final CollectionIoEvent<Id> entityIoEvent = new CollectionIoEvent<Id>(collection, entityId );
-
-
- final Iterator<MvccEntity> results = Collections.EMPTY_LIST.iterator();
-
- //mock up returning a list of MvccEntities
- when(serializationStrategy.load( collection, entityId, loadVersion, 1 )).thenReturn( results);
-
- Load load = new Load( uuidService, serializationStrategy, serializationFig );
- Entity loaded = load.call( entityIoEvent );
-
- assertNull( "No entity was loaded", loaded );
- }
-
-
-
-
-
- @Override
- protected void validateStage( final CollectionIoEvent<Id> event ) {
- final UUIDService uuidService = mock(UUIDService.class);
- final SerializationFig serializationFig = mock(SerializationFig.class);
- when(serializationFig.getBufferSize()).thenReturn( 10 );
-
-
- /**
- * Mock up wrong UUID type
- */
- when(uuidService.newTimeUUID()).thenReturn( UUID.randomUUID() );
-
-
- final MvccEntitySerializationStrategy serializationStrategy = mock(MvccEntitySerializationStrategy.class);
-
- new Load(uuidService, serializationStrategy, serializationFig).call( event );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
index f4a75d4..d1b2961 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
@@ -38,8 +38,12 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -53,7 +57,7 @@ public class EntityRepairImplTest {
* Tests changing from a full version to 2 updates, ensures we have a proper output
*/
@Test
- public void testSimpleRolling() {
+ public void testSimpleRolling() throws ConnectionException {
final SerializationFig serializationFig = mock( SerializationFig.class );
@@ -101,6 +105,11 @@ public class EntityRepairImplTest {
.thenReturn( Arrays.<MvccEntity>asList( v3, v2, v1 ).iterator() );
+ final MutationBatch mutationBatch = mock( MutationBatch.class);
+
+ when(mvccEntitySerializationStrategy.write( scope, v3 )).thenReturn( mutationBatch );
+
+
EntityRepairImpl entityRepair = new EntityRepairImpl( mvccEntitySerializationStrategy, serializationFig );
final MvccEntity returned = entityRepair.maybeRepair( scope, v3 );
@@ -118,12 +127,12 @@ public class EntityRepairImplTest {
final Entity finalVersion = returned.getEntity().get();
-
- final Object expectedField1Value = v2.getEntity().get().getField( "field1" ).getValue();
-
- final Object returnedField1Value = finalVersion.getField( "field1" ).getValue();
-
- assertEquals( "Same field value", expectedField1Value, returnedField1Value );
+//
+// final Object expectedField1Value = v2.getEntity().get().getField( "field1" ).getValue();
+//
+// final Object returnedField1Value = finalVersion.getField( "field1" ).getValue();
+//
+// assertEquals( "Same field value", expectedField1Value, returnedField1Value );
@@ -132,5 +141,7 @@ public class EntityRepairImplTest {
final Object returnedField2Value = finalVersion.getField( "field2" ).getValue();
assertEquals( "Same field value", expectedField2Value, returnedField2Value );
+
+ verify(mutationBatch).execute();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f1a88aa9/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index ba5ebce..8965668 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -163,7 +163,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now load it back
- MvccEntity returned = serializationStrategy.load( context, id, version );
+ MvccEntity returned = serializationStrategy.load( context, Collections.singleton( id), version ).getEntity( id );
assertEquals( "Mvcc entities are the same", saved, returned );
@@ -223,7 +223,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now get it, should be gone
- returned = serializationStrategy.load( context, id, version );
+ returned = serializationStrategy.load( context, Collections.singleton( id), version ).getEntity( id );
assertNull( returned );
}
@@ -255,7 +255,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now load it back
- MvccEntity returned = serializationStrategy.load( context, entityId, version );
+ MvccEntity returned = serializationStrategy.load( context, Collections.singleton( entityId ), version ).getEntity( entityId );
assertEquals( "Mvcc entities are the same", saved, returned );
@@ -270,7 +270,7 @@ public class MvccEntitySerializationStrategyImplTest {
serializationStrategy.mark( context, entityId, version ).execute();
- returned = serializationStrategy.load( context, entityId, version );
+ returned = serializationStrategy.load( context, Collections.singleton( entityId ), version ).getEntity( entityId );
assertEquals( entityId, returned.getId() );
assertEquals( version, returned.getVersion() );
@@ -281,7 +281,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now get it, should be gone
- returned = serializationStrategy.load( context, entityId, version );
+ returned = serializationStrategy.load( context, Collections.singleton( entityId ), version ).getEntity( entityId );
assertNull( returned );
}
@@ -318,7 +318,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now load it back
- MvccEntity returned = serializationStrategy.load( context, id, version );
+ MvccEntity returned = serializationStrategy.load( context, Collections.singleton( id ), version ).getEntity( id );
assertEquals( "Mvcc entities are the same", saved, returned );
@@ -349,7 +349,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now get it, should be gone
- returned = serializationStrategy.load( context, id, version );
+ returned = serializationStrategy.load( context, Collections.singleton( id ) , version ).getEntity( id );
assertNull( returned );
}
@@ -383,7 +383,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now load it back
- MvccEntity returnedV1 = serializationStrategy.load( context, id, version1 );
+ MvccEntity returnedV1 = serializationStrategy.load( context, Collections.singleton( id ) , version1 ).getEntity( id );
assertEquals( "Mvcc entities are the same", saved, returnedV1 );
@@ -403,7 +403,7 @@ public class MvccEntitySerializationStrategyImplTest {
serializationStrategy.write( context, savedV2 ).execute();
- MvccEntity returnedV2 = serializationStrategy.load( context, id, version2 );
+ MvccEntity returnedV2 = serializationStrategy.load( context, Collections.singleton( id ) , version2 ).getEntity( id );
assertEquals( "Mvcc entities are the same", savedV2, returnedV2 );
@@ -412,14 +412,14 @@ public class MvccEntitySerializationStrategyImplTest {
UUID version3 = UUIDGenerator.newTimeUUID();
- serializationStrategy.mark( context, id, version3 ).execute();
+ serializationStrategy.mark( context, id , version3 ).execute();
final Optional<Entity> empty = Optional.absent();
MvccEntity clearedV3 = new MvccEntityImpl( id, version3, MvccEntity.Status.COMPLETE, empty );
- MvccEntity returnedV3 = serializationStrategy.load( context, id, version3 );
+ MvccEntity returnedV3 = serializationStrategy.load( context, Collections.singleton( id ) , version3 ).getEntity( id );
assertEquals( "entities are the same", clearedV3, returnedV3 );
@@ -443,8 +443,8 @@ public class MvccEntitySerializationStrategyImplTest {
//now delete v2 and v1, we should still get v3
- serializationStrategy.delete( context, id, version1 ).execute();
- serializationStrategy.delete( context, id, version2 ).execute();
+ serializationStrategy.delete( context, id , version1 ).execute();
+ serializationStrategy.delete( context, id , version2 ).execute();
entities = serializationStrategy.load( context, id, current, 3 );
@@ -453,7 +453,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now get it, should be gone
- serializationStrategy.delete( context, id, version3 ).execute();
+ serializationStrategy.delete( context, id , version3 ).execute();
entities = serializationStrategy.load( context, id, current, 3 );
@@ -540,7 +540,7 @@ public class MvccEntitySerializationStrategyImplTest {
//now load it back
- MvccEntity returned = serializationStrategy.load( context, id, version );
+ MvccEntity returned = serializationStrategy.load( context, Collections.singleton( id ) , version ).getEntity( id );
assertEquals( "Mvcc entities are the same", saved, returned );
@@ -592,11 +592,11 @@ public class MvccEntitySerializationStrategyImplTest {
//now delete it
- serializationStrategy.delete( context, id, version ).execute();
+ serializationStrategy.delete( context, id , version ).execute();
//now get it, should be gone
- returned = serializationStrategy.load( context, id, version );
+ returned = serializationStrategy.load( context, Collections.singleton( id ) , version ).getEntity( id );
assertNull( returned );
}
@@ -641,7 +641,7 @@ public class MvccEntitySerializationStrategyImplTest {
@Test(expected = NullPointerException.class)
public void loadParamContext() throws ConnectionException {
- serializationStrategy.load( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID() );
+ serializationStrategy.load( null, Collections.<Id>emptyList(), UUIDGenerator.newTimeUUID() );
}
@@ -649,7 +649,7 @@ public class MvccEntitySerializationStrategyImplTest {
public void loadParamEntityId() throws ConnectionException {
serializationStrategy
- .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), (Id)null, UUIDGenerator.newTimeUUID() );
+ .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), null, UUIDGenerator.newTimeUUID() );
}
@@ -657,7 +657,7 @@ public class MvccEntitySerializationStrategyImplTest {
public void loadParamVersion() throws ConnectionException {
serializationStrategy
- .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), new SimpleId( "test" ), null );
+ .load( new CollectionScopeImpl(new SimpleId( "organization" ), new SimpleId( "test" ), "test" ), Collections.<Id>singleton( new SimpleId( "test" )), null );
}
[09/16] git commit: Fixes implementation of observables to be lazy
load
Posted by to...@apache.org.
Fixes implementation of observables to be lazy load
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b3515f45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b3515f45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b3515f45
Branch: refs/heads/two-dot-o
Commit: b3515f45cdfdcc0eedd6e667701f69eccad37a7d
Parents: 7af7e70
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 13:36:12 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 13:36:12 2014 -0600
----------------------------------------------------------------------
.../impl/EntityCollectionManagerImpl.java | 45 ++++++++++++++++----
1 file changed, 37 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b3515f45/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index c47bf1a..32f214f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -59,6 +59,7 @@ import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
import rx.Observable;
+import rx.Subscriber;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
@@ -212,10 +213,25 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
- final EntitySet results =
- entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
+ return Observable.create( new Observable.OnSubscribe<EntitySet>() {
+
+ @Override
+ public void call( final Subscriber<? super EntitySet> subscriber ) {
+ try {
+ final EntitySet results =
+ entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
+
+ subscriber.onNext( results );
+ subscriber.onCompleted();
+ }
+ catch ( Exception e ) {
+ subscriber.onError( e );
+ }
+ }
+ } );
+
+
- return Observable.just( results );
}
@@ -284,13 +300,26 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
- public Observable<VersionSet> getLatestVersion(
- Collection<Id> entityIds ) {
+ public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
+
+ return Observable.create( new Observable.OnSubscribe<VersionSet>() {
+
+ @Override
+ public void call( final Subscriber<? super VersionSet> subscriber ) {
+ try {
+ final VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
+ UUIDGenerator.newTimeUUID() );
+
+ subscriber.onNext( logEntries );
+ subscriber.onCompleted();
+ }
+ catch ( Exception e ) {
+ subscriber.onError( e );
+ }
+ }
+ } );
- VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
- UUIDGenerator.newTimeUUID() );
- return Observable.just(logEntries);
}
}
[08/16] git commit: adding method for get fields
Posted by to...@apache.org.
adding method for get fields
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c8fe1564
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c8fe1564
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c8fe1564
Branch: refs/heads/two-dot-o
Commit: c8fe1564fded77cf6f47bc0e8d6d28dcc21f0a30
Parents: 63094ab
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 9 13:29:20 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 9 13:29:20 2014 -0600
----------------------------------------------------------------------
.../impl/EntityCollectionManagerImpl.java | 20 ++++++++++++++++----
1 file changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c8fe1564/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 5fe2fbb..61f436f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -19,9 +19,10 @@
package org.apache.usergrid.persistence.collection.impl;
-import java.util.Collection;
-import java.util.Collections;
+import java.util.*;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +55,6 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import java.util.UUID;
import rx.Observable;
import rx.functions.Action1;
@@ -83,7 +83,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final WriteOptimisticVerify writeOptimisticVerify;
private final WriteCommit writeCommit;
private final RollbackAction rollback;
- private final GetVersion getVersion;
//delete stages
@@ -93,6 +92,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final TaskExecutor taskExecutor;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
+ private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
@Inject
public EntityCollectionManagerImpl( final UUIDService uuidService, @Write final WriteStart writeStart,
@@ -102,9 +102,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final WriteCommit writeCommit, final RollbackAction rollback,
final MarkStart markStart, final MarkCommit markCommit,
final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
@Assisted final CollectionScope collectionScope
) {
+ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ this.entitySerializationStrategy = entitySerializationStrategy;
Preconditions.checkNotNull( uuidService, "uuidService must be defined" );
@@ -217,6 +220,15 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
return Observable.just( results );
}
+ @Override
+ public Observable<Id> getIdField(Field field){
+ List<Field> fields = new ArrayList<>(1);
+ fields.add(field);
+ List list = Collections.singletonList(field);
+ rx.Observable.from(list).su
+ return null;
+ }
+
@Override
public Observable<Entity> update( final Entity entity ) {
[16/16] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
collection_multiget
Posted by to...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into collection_multiget
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/GetVersion.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a336b6b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a336b6b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a336b6b4
Branch: refs/heads/two-dot-o
Commit: a336b6b40ddfc4e39c6b92a0516dba0f626bc0d3
Parents: 454ef4f 8b0355f
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 16:03:40 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 16:03:40 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 3 -
.../corepersistence/StaleIndexCleanupTest.java | 51 +++---
.../index/impl/EsEntityIndexBatchImpl.java | 119 ++++++--------
.../index/impl/EsEntityIndexImpl.java | 154 +++++++------------
.../persistence/index/impl/IndexingUtils.java | 66 ++++++++
5 files changed, 199 insertions(+), 194 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a336b6b4/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a336b6b4/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
[12/16] git commit: fix merge
Posted by to...@apache.org.
fix merge
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f43e4f5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f43e4f5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f43e4f5c
Branch: refs/heads/two-dot-o
Commit: f43e4f5c3edc634f9cb6ab6b32d0f31a5f5efb0a
Parents: a61e146
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 9 14:01:25 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 9 14:01:25 2014 -0600
----------------------------------------------------------------------
.../collection/impl/EntityCollectionManagerImpl.java | 12 +-----------
1 file changed, 1 insertion(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f43e4f5c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index a1b9a13..c57f666 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -325,13 +325,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
@Override
-<<<<<<< HEAD
- public Observable<VersionSet> getLatestVersion(
- Collection<Id> entityIds) {
- VersionSet logEntries = mvccLogEntrySerializationStrategy.load(collectionScope, entityIds,
- UUIDGenerator.newTimeUUID());
-=======
public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
return Observable.create( new Observable.OnSubscribe<VersionSet>() {
@@ -349,10 +343,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
subscriber.onError( e );
}
}
- } );
-
->>>>>>> b3515f45cdfdcc0eedd6e667701f69eccad37a7d
-
-
+ } );
}
}
[13/16] Refactored loader to group for batching,
then call filters to perform the load, verification and result construction
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java
new file mode 100644
index 0000000..feed396
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpEntityMapUtils.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License. For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+
+package org.apache.usergrid.corepersistence.util;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.ArrayField;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.ByteArrayField;
+import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
+import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.FloatField;
+import org.apache.usergrid.persistence.model.field.IntegerField;
+import org.apache.usergrid.persistence.model.field.ListField;
+import org.apache.usergrid.persistence.model.field.LocationField;
+import org.apache.usergrid.persistence.model.field.LongField;
+import org.apache.usergrid.persistence.model.field.SetField;
+import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.field.UUIDField;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
+import org.apache.usergrid.persistence.model.field.value.Location;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Utilities for converting entities to/from maps suitable for Core Persistence.
+ * Aware of unique properties via Schema.
+ */
+public class CpEntityMapUtils {
+ private static final Logger logger = LoggerFactory.getLogger( CpEntityMapUtils.class );
+
+ public static ObjectMapper objectMapper = new ObjectMapper( );
+
+ public static Entity fromMap( Map<String, Object> map, String entityType, boolean topLevel ) {
+ return fromMap( null, map, entityType, topLevel );
+ }
+
+ public static Entity fromMap( Entity entity, Map<String, Object> map, String entityType, boolean topLevel ) {
+
+ if ( entity == null ) {
+ entity = new Entity();
+ }
+
+ for ( String fieldName : map.keySet() ) {
+
+ Object value = map.get( fieldName );
+ boolean unique = Schema.getDefaultSchema().isPropertyUnique(entityType, fieldName);
+
+// if ( unique ) {
+// logger.debug("{} is a unique property", fieldName );
+// }
+
+ if ( value instanceof String ) {
+ entity.setField( new StringField( fieldName, (String)value, unique && topLevel ));
+
+ } else if ( value instanceof Boolean ) {
+ entity.setField( new BooleanField( fieldName, (Boolean)value, unique && topLevel ));
+
+ } else if ( value instanceof Integer ) {
+ entity.setField( new IntegerField( fieldName, (Integer)value, unique && topLevel ));
+
+ } else if ( value instanceof Double ) {
+ entity.setField( new DoubleField( fieldName, (Double)value, unique && topLevel ));
+
+ } else if ( value instanceof Float ) {
+ entity.setField( new FloatField( fieldName, (Float)value, unique && topLevel ));
+
+ } else if ( value instanceof Long ) {
+ entity.setField( new LongField( fieldName, (Long)value, unique && topLevel ));
+
+ } else if ( value instanceof List) {
+ entity.setField( listToListField( fieldName, (List)value, entityType ));
+
+ } else if ( value instanceof UUID) {
+ entity.setField( new UUIDField( fieldName, (UUID)value, unique && topLevel ));
+
+ } else if ( value instanceof Map ) {
+ processMapValue( value, fieldName, entity, entityType);
+
+ } else if ( value instanceof Enum ) {
+ entity.setField( new StringField( fieldName, value.toString(), unique && topLevel ));
+
+ } else if ( value != null ) {
+ byte[] valueSerialized;
+ try {
+ valueSerialized = objectMapper.writeValueAsBytes( value );
+ }
+ catch ( JsonProcessingException e ) {
+ throw new RuntimeException( "Can't serialize object ",e );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Can't serialize object ",e );
+ }
+ ByteBuffer byteBuffer = ByteBuffer.wrap( valueSerialized );
+ ByteArrayField bf = new ByteArrayField( fieldName, byteBuffer.array(), value.getClass() );
+ entity.setField( bf );
+ }
+ }
+
+ return entity;
+ }
+
+ private static void processMapValue(
+ Object value, String fieldName, Entity entity, String entityType) {
+
+ Field field = null;
+
+ // is the map really a location element?
+ Map<String, Object> m = (Map<String, Object>)value;
+ if ( m.size() == 2) {
+ Double lat = null;
+ Double lon = null;
+ try {
+ if ( m.get("latitude") != null && m.get("longitude") != null ) {
+ lat = Double.parseDouble( m.get("latitude").toString() );
+ lon = Double.parseDouble( m.get("longitude").toString() );
+
+ } else if ( m.get("lat") != null && m.get("lon") != null ) {
+ lat = Double.parseDouble( m.get("lat").toString() );
+ lon = Double.parseDouble( m.get("lon").toString() );
+ }
+ } catch ( NumberFormatException ignored ) {}
+
+ if ( lat != null && lon != null ) {
+ field = new LocationField( fieldName, new Location( lat, lon ));
+ }
+ }
+
+ if ( field == null ) {
+
+ // not a location element, process it as map
+ entity.setField( new EntityObjectField( fieldName,
+ fromMap( (Map<String, Object>)value, entityType, false ))); // recursion
+
+ } else {
+ entity.setField( field );
+ }
+ }
+
+
+ private static ListField listToListField( String fieldName, List list, String entityType ) {
+
+ if (list.isEmpty()) {
+ return new ListField( fieldName );
+ }
+
+ Object sample = list.get(0);
+
+ if ( sample instanceof Map ) {
+ return new ListField<Entity>( fieldName, processListForField( list, entityType ));
+
+ } else if ( sample instanceof List ) {
+ return new ListField<List>( fieldName, processListForField( list, entityType ));
+
+ } else if ( sample instanceof String ) {
+ return new ListField<String>( fieldName, (List<String>)list );
+
+ } else if ( sample instanceof Boolean ) {
+ return new ListField<Boolean>( fieldName, (List<Boolean>)list );
+
+ } else if ( sample instanceof Integer ) {
+ return new ListField<Integer>( fieldName, (List<Integer>)list );
+
+ } else if ( sample instanceof Double ) {
+ return new ListField<Double>( fieldName, (List<Double>)list );
+
+ } else if ( sample instanceof Long ) {
+ return new ListField<Long>( fieldName, (List<Long>)list );
+
+ } else {
+ throw new RuntimeException("Unknown type " + sample.getClass().getName());
+ }
+ }
+
+
+ private static List processListForField( List list, String entityType ) {
+ if ( list.isEmpty() ) {
+ return list;
+ }
+ Object sample = list.get(0);
+
+ if ( sample instanceof Map ) {
+ List<Entity> newList = new ArrayList<Entity>();
+ for ( Map<String, Object> map : (List<Map<String, Object>>)list ) {
+ newList.add( fromMap( map, entityType, false ) );
+ }
+ return newList;
+
+ } else if ( sample instanceof List ) {
+ return processListForField( list, entityType ); // recursion
+
+ } else {
+ return list;
+ }
+ }
+
+
+ /**
+ * Convert Entity to Map, adding version_ug_field and a {name}_ug_analyzed field for each
+ * StringField.
+ */
+ public static Map toMap(EntityObject entity) {
+
+ Map<String, Object> entityMap = new TreeMap<>();
+
+ for (Object f : entity.getFields().toArray()) {
+ Field field = (Field) f;
+
+ if (f instanceof ListField || f instanceof ArrayField) {
+ List list = (List) field.getValue();
+ entityMap.put(field.getName(),
+ new ArrayList( processCollectionForMap(list)));
+
+ } else if (f instanceof SetField) {
+ Set set = (Set) field.getValue();
+ entityMap.put(field.getName(),
+ new ArrayList( processCollectionForMap(set)));
+
+ } else if (f instanceof EntityObjectField) {
+ EntityObject eo = (EntityObject) field.getValue();
+ entityMap.put( field.getName(), toMap(eo)); // recursion
+
+ } else if (f instanceof StringField) {
+ entityMap.put(field.getName(), ((String) field.getValue()));
+
+ } else if (f instanceof LocationField) {
+ LocationField locField = (LocationField) f;
+ Map<String, Object> locMap = new HashMap<String, Object>();
+
+ // field names lat and lon trigger ElasticSearch geo location
+ locMap.put("lat", locField.getValue().getLatitude());
+ locMap.put("lon", locField.getValue().getLongitude());
+ entityMap.put( field.getName(), field.getValue());
+
+ } else if (f instanceof ByteArrayField) {
+ ByteArrayField bf = ( ByteArrayField ) f;
+
+ byte[] serilizedObj = bf.getValue();
+ Object o;
+ try {
+ o = objectMapper.readValue( serilizedObj, bf.getClassinfo() );
+ }
+ catch ( IOException e ) {
+ throw new RuntimeException( "Can't deserialize object ",e );
+ }
+ entityMap.put( bf.getName(), o );
+ }
+ else {
+ entityMap.put( field.getName(), field.getValue());
+ }
+ }
+
+ return entityMap;
+ }
+
+
+ private static Collection processCollectionForMap(Collection c) {
+ if (c.isEmpty()) {
+ return c;
+ }
+ List processed = new ArrayList();
+ Object sample = c.iterator().next();
+
+ if (sample instanceof Entity) {
+ for (Object o : c.toArray()) {
+ Entity e = (Entity) o;
+ processed.add(toMap(e));
+ }
+
+ } else if (sample instanceof List) {
+ for (Object o : c.toArray()) {
+ List list = (List) o;
+ processed.add(processCollectionForMap(list)); // recursion;
+ }
+
+ } else if (sample instanceof Set) {
+ for (Object o : c.toArray()) {
+ Set set = (Set) o;
+ processed.add(processCollectionForMap(set)); // recursion;
+ }
+
+ } else {
+ for (Object o : c.toArray()) {
+ processed.add(o);
+ }
+ }
+ return processed;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
new file mode 100644
index 0000000..f2fce47
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -0,0 +1,106 @@
+package org.apache.usergrid.corepersistence.util;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.Schema;
+
+
+/**
+ * Utilises for constructing standard naming conventions for collections and connections
+ */
+public class CpNamingUtils {
+
+ /**
+ * Edge types for all types
+ */
+ public static final String ALL_TYPES = "zzzalltypeszzz";
+
+ /**
+ * Edge types for collection suffix
+ */
+ public static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
+
+ /**
+ * Edge types for connection suffix
+ */
+ public static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
+
+
+ public static String getCollectionScopeNameFromEntityType( String type ) {
+ String csn = EDGE_COLL_SUFFIX + Schema.defaultCollectionName( type );
+ return csn.toLowerCase();
+ }
+
+
+ public static String getCollectionScopeNameFromCollectionName( String name ) {
+ String csn = EDGE_COLL_SUFFIX + name;
+ return csn.toLowerCase();
+ }
+
+
+ public static String getConnectionScopeName( String entityType, String connectionType ) {
+ String csn = EDGE_CONN_SUFFIX + connectionType + entityType;
+ return csn.toLowerCase();
+ }
+
+
+ public static boolean isCollectionEdgeType( String type ) {
+ return type.startsWith( EDGE_COLL_SUFFIX );
+ }
+
+
+ public static boolean isConnectionEdgeType( String type ) {
+ return type.startsWith( EDGE_CONN_SUFFIX );
+ }
+
+
+ static public String getConnectionType( String edgeType ) {
+ String[] parts = edgeType.split( "\\|" );
+ return parts[1];
+ }
+
+
+ static public String getCollectionName( String edgeType ) {
+ String[] parts = edgeType.split( "\\|" );
+ return parts[1];
+ }
+
+
+ public static String getEdgeTypeFromConnectionType( String connectionType ) {
+
+ if ( connectionType != null ) {
+ String csn = EDGE_CONN_SUFFIX + "|" + connectionType;
+ return csn;
+ }
+
+ return null;
+ }
+
+
+ public static String getEdgeTypeFromCollectionName( String collectionName ) {
+
+ if ( collectionName != null ) {
+ String csn = EDGE_COLL_SUFFIX + "|" + collectionName;
+ return csn;
+ }
+
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
index 87f6b46..a75b9fd 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityMapUtilsTest.java
@@ -25,6 +25,8 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.ListField;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index c5d5782..5a1eca2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.UUID;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.usergrid.AbstractCoreIT;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
[10/16] git commit: merge and get fielded
Posted by to...@apache.org.
merge and get fielded
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0d508f98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0d508f98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0d508f98
Branch: refs/heads/two-dot-o
Commit: 0d508f981db9d509ef3250cbf75cc2f5384dd491
Parents: c8fe156 7af7e70
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 9 13:57:49 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 9 13:57:49 2014 -0600
----------------------------------------------------------------------
.../collection/EntityCollectionManager.java | 16 +-
.../persistence/collection/EntitySet.java | 2 -
.../persistence/collection/MvccEntity.java | 68 ++++++
.../persistence/collection/MvccLogEntry.java | 94 +++++++++
.../persistence/collection/VersionSet.java | 67 ++++++
.../exception/CollectionRuntimeException.java | 2 +-
.../exception/WriteCommitException.java | 4 +-
.../WriteOptimisticVerifyException.java | 4 +-
.../exception/WriteStartException.java | 4 +-
.../exception/WriteUniqueVerifyException.java | 2 +-
.../collection/guice/CollectionModule.java | 2 +-
.../impl/EntityCollectionManagerImpl.java | 211 ++++++++++---------
.../impl/EntityVersionCleanupTask.java | 2 +-
.../mvcc/MvccEntitySerializationStrategy.java | 3 +-
.../mvcc/MvccLogEntrySerializationStrategy.java | 17 +-
.../mvcc/changelog/ChangeLogGenerator.java | 5 +-
.../mvcc/changelog/ChangeLogGeneratorImpl.java | 2 +-
.../collection/mvcc/entity/MvccEntity.java | 68 ------
.../collection/mvcc/entity/MvccLogEntry.java | 93 --------
.../mvcc/entity/MvccValidationUtils.java | 1 +
.../mvcc/entity/impl/MvccEntityDeleteEvent.java | 2 +-
.../mvcc/entity/impl/MvccEntityEvent.java | 2 +-
.../mvcc/entity/impl/MvccEntityImpl.java | 2 +-
.../mvcc/entity/impl/MvccEntityWriteEvent.java | 3 +-
.../mvcc/entity/impl/MvccLogEntryImpl.java | 2 +-
.../mvcc/stage/EntityUpdateEvent.java | 1 -
.../mvcc/stage/delete/MarkCommit.java | 4 +-
.../collection/mvcc/stage/delete/MarkStart.java | 4 +-
.../mvcc/stage/write/RollbackAction.java | 2 +-
.../mvcc/stage/write/WriteCommit.java | 4 +-
.../mvcc/stage/write/WriteOptimisticVerify.java | 44 ++--
.../collection/mvcc/stage/write/WriteStart.java | 5 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../collection/serialization/EntityRepair.java | 2 +-
.../serialization/OptimisticUpdate.java | 2 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../serialization/impl/EntitySetImpl.java | 3 +-
.../serialization/impl/LogEntryIterator.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 3 +-
.../MvccLogEntrySerializationStrategyImpl.java | 129 ++++++++----
.../serialization/impl/VersionSetImpl.java | 80 +++++++
.../collection/EntityCollectionManagerIT.java | 155 +++++++++-----
.../changelog/ChangeLogGeneratorImplTest.java | 3 +-
.../mvcc/entity/impl/MvccEntityImplTest.java | 2 +-
.../mvcc/entity/impl/MvccLogEntryImplTest.java | 2 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 2 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 6 +-
.../mvcc/stage/delete/MarkStartTest.java | 4 +-
.../mvcc/stage/write/WriteCommitTest.java | 4 +-
.../stage/write/WriteOptimisticVerifyTest.java | 4 +-
.../mvcc/stage/write/WriteStartTest.java | 4 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../serialization/EntityRepairImplTest.java | 2 +-
.../impl/LogEntryIteratorTest.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 2 +-
.../impl/MvccLESSTransientTest.java | 7 +-
...ccLogEntrySerializationStrategyImplTest.java | 19 +-
.../util/InvalidMvccEntityGenerator.java | 2 +-
.../util/InvalidValueGeneratorTest.java | 2 +-
.../collection/util/LogEntryMock.java | 4 +-
61 files changed, 724 insertions(+), 473 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0d508f98/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 2625078,41d19e6..f976cb5
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@@ -25,6 -25,6 +25,7 @@@ import java.util.UUID
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
++import org.apache.usergrid.persistence.model.field.Field;
import rx.Observable;
@@@ -55,12 -55,11 +56,18 @@@ public interface EntityCollectionManage
public Observable<Entity> load( Id entityId );
/**
- * Return the latest version of the specified entity.
+ * Return the latest versions of the specified entityIds
*/
- public Observable<UUID> getLatestVersion( Id entityId );
- //TODO TN Change load to use multiget and return multiple entities. Only supports loading 1k per load operation.
+ public Observable<VersionSet> getLatestVersion( Collection<Id> entityId );
+
+ /**
++ * Gets the Id for a field
++ * @param field
++ * @return most likely a single Id, watch for onerror events
++ */
++ public Observable<Id> getIdField(final Field field);
+
+ /**
* Load all the entityIds into the observable entity set
* @param entityIds
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0d508f98/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 61f436f,c47bf1a..fab4269
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -19,10 -19,11 +19,18 @@@
package org.apache.usergrid.persistence.collection.impl;
++import java.net.ConnectException;
+import java.util.*;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.List;
-import java.util.UUID;
++import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
++import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
++import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
++import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -33,7 -35,9 +42,8 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
- import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+ import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+ import org.apache.usergrid.persistence.collection.MvccEntity;
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
@@@ -64,13 -66,12 +72,12 @@@ import rx.schedulers.Schedulers
/**
* Simple implementation. Should perform writes, delete and load.
-- *
++ * <p/>
* TODO: maybe refactor the stage operations into their own classes for clarity and organization?
*/
public class EntityCollectionManagerImpl implements EntityCollectionManager {
-- private static final Logger log = LoggerFactory.getLogger( EntityCollectionManagerImpl.class );
++ private static final Logger log = LoggerFactory.getLogger(EntityCollectionManagerImpl.class);
private final CollectionScope collectionScope;
private final UUIDService uuidService;
@@@ -91,28 -92,26 +98,31 @@@
private final TaskExecutor taskExecutor;
+ private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
+ private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+
@Inject
-- public EntityCollectionManagerImpl( final UUIDService uuidService, @Write final WriteStart writeStart,
-- @WriteUpdate final WriteStart writeUpdate,
-- final WriteUniqueVerify writeVerifyUnique,
-- final WriteOptimisticVerify writeOptimisticVerify,
-- final WriteCommit writeCommit, final RollbackAction rollback,
-- final MarkStart markStart, final MarkCommit markCommit,
-- final MvccEntitySerializationStrategy entitySerializationStrategy,
- final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
-- @CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope
- ) {
- final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
- @Assisted final CollectionScope collectionScope ) {
++ public EntityCollectionManagerImpl(final UUIDService uuidService, @Write final WriteStart writeStart,
++ @WriteUpdate final WriteStart writeUpdate,
++ final WriteUniqueVerify writeVerifyUnique,
++ final WriteOptimisticVerify writeOptimisticVerify,
++ final WriteCommit writeCommit, final RollbackAction rollback,
++ final MarkStart markStart, final MarkCommit markCommit,
++ final MvccEntitySerializationStrategy entitySerializationStrategy,
++ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
++ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
++ @CollectionTaskExecutor final TaskExecutor taskExecutor,
++ @Assisted final CollectionScope collectionScope
++ ) {
+ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ this.entitySerializationStrategy = entitySerializationStrategy;
-- Preconditions.checkNotNull( uuidService, "uuidService must be defined" );
++ Preconditions.checkNotNull(uuidService, "uuidService must be defined");
-- MvccValidationUtils.validateCollectionScope( collectionScope );
++ MvccValidationUtils.validateCollectionScope(collectionScope);
this.writeStart = writeStart;
this.writeUpdate = writeUpdate;
@@@ -129,175 -127,170 +138,189 @@@
this.uuidService = uuidService;
this.collectionScope = collectionScope;
this.taskExecutor = taskExecutor;
- this.entitySerializationStrategy = entitySerializationStrategy;
+ this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
}
@Override
-- public Observable<Entity> write( final Entity entity ) {
++ public Observable<Entity> write(final Entity entity) {
//do our input validation
- Preconditions.checkNotNull( entity,
- "Entity is required in the new stage of the mvcc write" );
- Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
++ Preconditions.checkNotNull(entity, "Entity is required in the new stage of the mvcc write");
final Id entityId = entity.getId();
-- ValidationUtils.verifyIdentity( entityId );
++ ValidationUtils.verifyIdentity(entityId);
// create our observable and start the write
-- final CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
++ final CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>(collectionScope, entity);
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData,writeStart );
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeStart );
++ Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner(writeData, writeStart);
// execute all validation stages concurrently. Needs refactored when this is done.
// https://github.com/Netflix/RxJava/issues/627
// observable = Concurrent.concurrent( observable, Schedulers.io(), new WaitZip(),
// writeVerifyUnique, writeOptimisticVerify );
-- observable.map( writeCommit ).doOnNext( new Action1<Entity>() {
++ observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
-- public void call( final Entity entity ) {
++ public void call(final Entity entity) {
//TODO fire a task here
//post-processing to come later. leave it empty for now.
}
-- } ).doOnError( rollback );
++ }).doOnError(rollback);
// return the commit result.
-- return observable.map( writeCommit ).doOnError( rollback );
++ return observable.map(writeCommit).doOnError(rollback);
}
@Override
-- public Observable<Void> delete( final Id entityId ) {
-
- Preconditions.checkNotNull( entityId, "Entity id is required in this stage" );
- Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
- Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
-
- return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) ).map( markStart )
- .doOnNext( markCommit ).map( new Func1<CollectionIoEvent<MvccEntity>, Void>() {
- @Override
- public Void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
- return null;
- }
- } );
++ public Observable<Void> delete(final Id entityId) {
+
- Preconditions.checkNotNull( entityId, "Entity id is required in this stage" );
- Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
- Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
++ Preconditions.checkNotNull(entityId, "Entity id is required in this stage");
++ Preconditions.checkNotNull(entityId.getUuid(), "Entity id is required in this stage");
++ Preconditions.checkNotNull(entityId.getType(), "Entity type is required in this stage");
+
- return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) )
- .map( markStart ).doOnNext( markCommit ).map( new Func1<CollectionIoEvent<MvccEntity>,
- Void>() {
++ return Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId)).map(markStart)
++ .doOnNext(markCommit).map(new Func1<CollectionIoEvent<MvccEntity>, Void>() {
+ @Override
- public Void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
++ public Void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
+ return null;
+ }
- } );
++ });
}
@Override
-- public Observable<Entity> load( final Id entityId ) {
++ public Observable<Entity> load(final Id entityId) {
-- Preconditions.checkNotNull( entityId, "Entity id required in the load stage" );
-- Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
-- Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
++ Preconditions.checkNotNull(entityId, "Entity id required in the load stage");
++ Preconditions.checkNotNull(entityId.getUuid(), "Entity id uuid required in load stage");
++ Preconditions.checkNotNull(entityId.getType(), "Entity id type required in load stage");
- return load( Collections.singleton(entityId) ).map( new Func1<EntitySet, Entity>() {
- return load( Collections.singleton( entityId ) ).map( new Func1<EntitySet, Entity>() {
++ return load(Collections.singleton(entityId)).map(new Func1<EntitySet, Entity>() {
@Override
-- public Entity call( final EntitySet entitySet ) {
-- final MvccEntity entity = entitySet.getEntity( entityId );
++ public Entity call(final EntitySet entitySet) {
++ final MvccEntity entity = entitySet.getEntity(entityId);
- if(entity == null){
- if ( entity == null ) {
++ if (entity == null) {
return null;
}
return entity.getEntity().orNull();
}
-- } );
++ });
}
@Override
-- public Observable<EntitySet> load( final Collection<Id> entityIds ) {
++ public Observable<EntitySet> load(final Collection<Id> entityIds) {
-- Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
++ Preconditions.checkNotNull(entityIds, "entityIds cannot be null");
- final EntitySet
- results = entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
+ final EntitySet results =
- entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
++ entitySerializationStrategy.load(collectionScope, entityIds, UUIDGenerator.newTimeUUID());
+
- return Observable.just( results );
++ return Observable.just(results);
+ }
- return Observable.just( results );
+ @Override
- public Observable<Id> getIdField(Field field){
- List<Field> fields = new ArrayList<>(1);
- fields.add(field);
- List list = Collections.singletonList(field);
- rx.Observable.from(list).su
- return null;
++ public Observable<Id> getIdField(final Field field) {
++ final List<Field> fields = Collections.singletonList(field);
++ rx.Observable<Id> o = rx.Observable.from(fields).map(new Func1<Field, Id>() {
++ @Override
++ public Id call(Field field) {
++ try {
++ UniqueValueSet set = uniqueValueSerializationStrategy.load(collectionScope, fields);
++ UniqueValue value = set.getValue(field.getName());
++ Id id = value == null ? null :value.getEntityId();
++ return id;
++ } catch (ConnectionException e) {
++ log.error("Failed to getIdField", e);
++ throw new RuntimeException(e);
++ }
++ }
++ });
++ return o;
}
@Override
-- public Observable<Entity> update( final Entity entity ) {
++ public Observable<Entity> update(final Entity entity) {
-- log.debug( "Starting update process" );
++ log.debug("Starting update process");
//do our input validation
-- Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
++ Preconditions.checkNotNull(entity, "Entity is required in the new stage of the mvcc write");
final Id entityId = entity.getId();
-- ValidationUtils.verifyIdentity( entityId );
++ ValidationUtils.verifyIdentity(entityId);
// create our observable and start the write
-- CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
++ CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>(collectionScope, entity);
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData,writeUpdate );
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
++ Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner(writeData, writeUpdate);
-- return observable.map( writeCommit ).doOnNext( new Action1<Entity>() {
++ return observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
-- public void call( final Entity entity ) {
-- log.debug( "sending entity to the queue" );
++ public void call(final Entity entity) {
++ log.debug("sending entity to the queue");
- //we an update, signal the fix
+ //we an update, signal the fix
//TODO T.N Change this to fire a task
- // Observable.from( new CollectionIoEvent<Id>(collectionScope, entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
+ // Observable.from( new CollectionIoEvent<Id>(collectionScope,
+ // entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
}
-- } ).doOnError( rollback );
++ }).doOnError(rollback);
}
- // fire the stages
- public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData, WriteStart writeState ) {
-
- return Observable.from( writeData ).map( writeState ).doOnNext( new Action1<CollectionIoEvent<MvccEntity>>() {
- @Override
- public void call(
- final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
+ // fire the stages
- public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData,
- WriteStart writeState ) {
++ public Observable<CollectionIoEvent<MvccEntity>> stageRunner(CollectionIoEvent<Entity> writeData,
++ WriteStart writeState) {
- Observable<CollectionIoEvent<MvccEntity>> unique =
- Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
- .doOnNext( writeVerifyUnique );
- return Observable.from( writeData ).map( writeState ).doOnNext( new Action1<CollectionIoEvent<MvccEntity>>() {
++ return Observable.from(writeData).map(writeState).doOnNext(new Action1<CollectionIoEvent<MvccEntity>>() {
+ @Override
- public void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
++ public void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
- // optimistic verification
- Observable<CollectionIoEvent<MvccEntity>> optimistic =
- Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
- .doOnNext( writeOptimisticVerify );
+ Observable<CollectionIoEvent<MvccEntity>> unique =
- Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
- .doOnNext( writeVerifyUnique );
++ Observable.from(mvccEntityCollectionIoEvent).subscribeOn(Schedulers.io())
++ .doOnNext(writeVerifyUnique);
- //wait for both to finish
- Observable.merge( unique, optimistic ).toBlocking().last();
+ // optimistic verification
+ Observable<CollectionIoEvent<MvccEntity>> optimistic =
- Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
- .doOnNext( writeOptimisticVerify );
++ Observable.from(mvccEntityCollectionIoEvent).subscribeOn(Schedulers.io())
++ .doOnNext(writeOptimisticVerify);
- }
- } );
+ //wait for both to finish
- Observable.merge( unique, optimistic ).toBlocking().last();
++ Observable.merge(unique, optimistic).toBlocking().last();
+ }
- } );
++ });
}
@Override
- public Observable<UUID> getLatestVersion(Id entityId) {
- return Observable.from(
- new CollectionIoEvent<Id>( collectionScope, entityId ) ).map(getVersion);
- }
+ public Observable<VersionSet> getLatestVersion(
- Collection<Id> entityIds ) {
++ Collection<Id> entityIds) {
+
- VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
- UUIDGenerator.newTimeUUID() );
++ VersionSet logEntries = mvccLogEntrySerializationStrategy.load(collectionScope, entityIds,
++ UUIDGenerator.newTimeUUID());
+
+ return Observable.just(logEntries);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0d508f98/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index a7f9ea2,c01e022..6afad33
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@@ -22,6 -23,6 +23,7 @@@ import java.util.Arrays
import java.util.List;
import java.util.UUID;
++import org.apache.usergrid.persistence.model.field.Field;
import org.jukito.UseModules;
import org.junit.Rule;
import org.junit.Test;
@@@ -277,6 -278,6 +279,36 @@@ public class EntityCollectionManagerIT
assertNotNull( collectionScope3 );
}
++ @Test
++ public void writeAndGetField() {
++
++
++ CollectionScope collectionScope1 =
++ new CollectionScopeImpl(new SimpleId("organization"), new SimpleId("test1"), "test1");
++
++ Entity newEntity = new Entity(new SimpleId("test"));
++ Field field = new StringField("testField", "unique", true);
++ newEntity.setField(field);
++
++ EntityCollectionManager manager = factory.createCollectionManager(collectionScope1);
++
++ Observable<Entity> observable = manager.write(newEntity);
++
++ Entity createReturned = observable.toBlocking().lastOrDefault(null);
++
++
++ assertNotNull("Id was assigned", createReturned.getId());
++ assertNotNull("Version was assigned", createReturned.getVersion());
++
++ Id id = manager.getIdField(field).toBlocking().lastOrDefault(null);
++ assertNotNull(id);
++
++ Field fieldNull = new StringField("testFieldNotThere", "uniquely", true);
++ id = manager.getIdField(fieldNull).toBlocking().lastOrDefault(null);
++ assertNull(id);
++
++ }
++
@Test
public void partialUpdate() {
[14/16] git commit: Refactored loader to group for batching,
then call filters to perform the load, verification and result construction
Posted by to...@apache.org.
Refactored loader to group for batching, then call filters to perform the load, verification and result construction
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6285e605
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6285e605
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6285e605
Branch: refs/heads/two-dot-o
Commit: 6285e6057832580d947106f845852254397eb56c
Parents: b3515f4
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 15:03:51 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 15:49:55 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 3 +
.../corepersistence/CpEntityManagerFactory.java | 2 +
.../corepersistence/CpEntityMapUtils.java | 323 -------------------
.../corepersistence/CpManagerCache.java | 2 +-
.../usergrid/corepersistence/CpNamingUtils.java | 106 ------
.../corepersistence/CpRelationManager.java | 140 +-------
.../results/AbstractIdLoader.java | 130 --------
.../corepersistence/results/EntitiesLoader.java | 51 ---
.../corepersistence/results/EntityVerifier.java | 142 ++++++++
.../results/FilteringLoader.java | 240 ++++++++++++++
.../corepersistence/results/IdsLoader.java | 46 ---
.../corepersistence/results/IdsVerifier.java | 65 ++++
.../corepersistence/results/RefsLoader.java | 46 ---
.../corepersistence/results/RefsVerifier.java | 62 ++++
.../corepersistence/results/ResultsLoader.java | 5 +-
.../results/ResultsLoaderFactory.java | 9 +-
.../results/ResultsLoaderFactoryImpl.java | 42 ++-
.../results/ResultsVerifier.java | 72 +++++
.../results/VersionVerifier.java | 103 ++++++
.../corepersistence/util/CpEntityMapUtils.java | 323 +++++++++++++++++++
.../corepersistence/util/CpNamingUtils.java | 106 ++++++
.../corepersistence/CpEntityMapUtilsTest.java | 2 +
.../corepersistence/StaleIndexCleanupTest.java | 1 +
23 files changed, 1173 insertions(+), 848 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 1a8f17c..657ee2d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -54,6 +54,9 @@ import me.prettyprint.hector.api.query.QueryResult;
import me.prettyprint.hector.api.query.SliceCounterQuery;
import static org.apache.commons.lang.StringUtils.capitalize;
import static org.apache.commons.lang.StringUtils.isBlank;
+
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AggregateCounter;
import org.apache.usergrid.persistence.AggregateCounterSet;
import org.apache.usergrid.persistence.CollectionRef;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f44eac3..17b33e4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -27,6 +27,8 @@ import java.util.Map;
import java.util.TreeMap;
import java.util.UUID;
import org.apache.commons.lang.StringUtils;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.AbstractEntity;
import org.apache.usergrid.persistence.DynamicEntity;
import org.apache.usergrid.persistence.Entity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java
deleted file mode 100644
index efbef0b..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityMapUtils.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
-
-package org.apache.usergrid.corepersistence;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.model.entity.Entity;
-import org.apache.usergrid.persistence.model.field.ArrayField;
-import org.apache.usergrid.persistence.model.field.BooleanField;
-import org.apache.usergrid.persistence.model.field.ByteArrayField;
-import org.apache.usergrid.persistence.model.field.DoubleField;
-import org.apache.usergrid.persistence.model.field.EntityObjectField;
-import org.apache.usergrid.persistence.model.field.Field;
-import org.apache.usergrid.persistence.model.field.FloatField;
-import org.apache.usergrid.persistence.model.field.IntegerField;
-import org.apache.usergrid.persistence.model.field.ListField;
-import org.apache.usergrid.persistence.model.field.LocationField;
-import org.apache.usergrid.persistence.model.field.LongField;
-import org.apache.usergrid.persistence.model.field.SetField;
-import org.apache.usergrid.persistence.model.field.StringField;
-import org.apache.usergrid.persistence.model.field.UUIDField;
-import org.apache.usergrid.persistence.model.field.value.EntityObject;
-import org.apache.usergrid.persistence.model.field.value.Location;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * Utilities for converting entities to/from maps suitable for Core Persistence.
- * Aware of unique properties via Schema.
- */
-public class CpEntityMapUtils {
- private static final Logger logger = LoggerFactory.getLogger( CpEntityMapUtils.class );
-
- public static ObjectMapper objectMapper = new ObjectMapper( );
-
- public static Entity fromMap( Map<String, Object> map, String entityType, boolean topLevel ) {
- return fromMap( null, map, entityType, topLevel );
- }
-
- public static Entity fromMap( Entity entity, Map<String, Object> map, String entityType, boolean topLevel ) {
-
- if ( entity == null ) {
- entity = new Entity();
- }
-
- for ( String fieldName : map.keySet() ) {
-
- Object value = map.get( fieldName );
- boolean unique = Schema.getDefaultSchema().isPropertyUnique(entityType, fieldName);
-
-// if ( unique ) {
-// logger.debug("{} is a unique property", fieldName );
-// }
-
- if ( value instanceof String ) {
- entity.setField( new StringField( fieldName, (String)value, unique && topLevel ));
-
- } else if ( value instanceof Boolean ) {
- entity.setField( new BooleanField( fieldName, (Boolean)value, unique && topLevel ));
-
- } else if ( value instanceof Integer ) {
- entity.setField( new IntegerField( fieldName, (Integer)value, unique && topLevel ));
-
- } else if ( value instanceof Double ) {
- entity.setField( new DoubleField( fieldName, (Double)value, unique && topLevel ));
-
- } else if ( value instanceof Float ) {
- entity.setField( new FloatField( fieldName, (Float)value, unique && topLevel ));
-
- } else if ( value instanceof Long ) {
- entity.setField( new LongField( fieldName, (Long)value, unique && topLevel ));
-
- } else if ( value instanceof List) {
- entity.setField( listToListField( fieldName, (List)value, entityType ));
-
- } else if ( value instanceof UUID) {
- entity.setField( new UUIDField( fieldName, (UUID)value, unique && topLevel ));
-
- } else if ( value instanceof Map ) {
- processMapValue( value, fieldName, entity, entityType);
-
- } else if ( value instanceof Enum ) {
- entity.setField( new StringField( fieldName, value.toString(), unique && topLevel ));
-
- } else if ( value != null ) {
- byte[] valueSerialized;
- try {
- valueSerialized = objectMapper.writeValueAsBytes( value );
- }
- catch ( JsonProcessingException e ) {
- throw new RuntimeException( "Can't serialize object ",e );
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Can't serialize object ",e );
- }
- ByteBuffer byteBuffer = ByteBuffer.wrap( valueSerialized );
- ByteArrayField bf = new ByteArrayField( fieldName, byteBuffer.array(), value.getClass() );
- entity.setField( bf );
- }
- }
-
- return entity;
- }
-
- private static void processMapValue(
- Object value, String fieldName, Entity entity, String entityType) {
-
- Field field = null;
-
- // is the map really a location element?
- Map<String, Object> m = (Map<String, Object>)value;
- if ( m.size() == 2) {
- Double lat = null;
- Double lon = null;
- try {
- if ( m.get("latitude") != null && m.get("longitude") != null ) {
- lat = Double.parseDouble( m.get("latitude").toString() );
- lon = Double.parseDouble( m.get("longitude").toString() );
-
- } else if ( m.get("lat") != null && m.get("lon") != null ) {
- lat = Double.parseDouble( m.get("lat").toString() );
- lon = Double.parseDouble( m.get("lon").toString() );
- }
- } catch ( NumberFormatException ignored ) {}
-
- if ( lat != null && lon != null ) {
- field = new LocationField( fieldName, new Location( lat, lon ));
- }
- }
-
- if ( field == null ) {
-
- // not a location element, process it as map
- entity.setField( new EntityObjectField( fieldName,
- fromMap( (Map<String, Object>)value, entityType, false ))); // recursion
-
- } else {
- entity.setField( field );
- }
- }
-
-
- private static ListField listToListField( String fieldName, List list, String entityType ) {
-
- if (list.isEmpty()) {
- return new ListField( fieldName );
- }
-
- Object sample = list.get(0);
-
- if ( sample instanceof Map ) {
- return new ListField<Entity>( fieldName, processListForField( list, entityType ));
-
- } else if ( sample instanceof List ) {
- return new ListField<List>( fieldName, processListForField( list, entityType ));
-
- } else if ( sample instanceof String ) {
- return new ListField<String>( fieldName, (List<String>)list );
-
- } else if ( sample instanceof Boolean ) {
- return new ListField<Boolean>( fieldName, (List<Boolean>)list );
-
- } else if ( sample instanceof Integer ) {
- return new ListField<Integer>( fieldName, (List<Integer>)list );
-
- } else if ( sample instanceof Double ) {
- return new ListField<Double>( fieldName, (List<Double>)list );
-
- } else if ( sample instanceof Long ) {
- return new ListField<Long>( fieldName, (List<Long>)list );
-
- } else {
- throw new RuntimeException("Unknown type " + sample.getClass().getName());
- }
- }
-
-
- private static List processListForField( List list, String entityType ) {
- if ( list.isEmpty() ) {
- return list;
- }
- Object sample = list.get(0);
-
- if ( sample instanceof Map ) {
- List<Entity> newList = new ArrayList<Entity>();
- for ( Map<String, Object> map : (List<Map<String, Object>>)list ) {
- newList.add( fromMap( map, entityType, false ) );
- }
- return newList;
-
- } else if ( sample instanceof List ) {
- return processListForField( list, entityType ); // recursion
-
- } else {
- return list;
- }
- }
-
-
- /**
- * Convert Entity to Map, adding version_ug_field and a {name}_ug_analyzed field for each
- * StringField.
- */
- public static Map toMap(EntityObject entity) {
-
- Map<String, Object> entityMap = new TreeMap<>();
-
- for (Object f : entity.getFields().toArray()) {
- Field field = (Field) f;
-
- if (f instanceof ListField || f instanceof ArrayField) {
- List list = (List) field.getValue();
- entityMap.put(field.getName(),
- new ArrayList( processCollectionForMap(list)));
-
- } else if (f instanceof SetField) {
- Set set = (Set) field.getValue();
- entityMap.put(field.getName(),
- new ArrayList( processCollectionForMap(set)));
-
- } else if (f instanceof EntityObjectField) {
- EntityObject eo = (EntityObject) field.getValue();
- entityMap.put( field.getName(), toMap(eo)); // recursion
-
- } else if (f instanceof StringField) {
- entityMap.put(field.getName(), ((String) field.getValue()));
-
- } else if (f instanceof LocationField) {
- LocationField locField = (LocationField) f;
- Map<String, Object> locMap = new HashMap<String, Object>();
-
- // field names lat and lon trigger ElasticSearch geo location
- locMap.put("lat", locField.getValue().getLatitude());
- locMap.put("lon", locField.getValue().getLongitude());
- entityMap.put( field.getName(), field.getValue());
-
- } else if (f instanceof ByteArrayField) {
- ByteArrayField bf = ( ByteArrayField ) f;
-
- byte[] serilizedObj = bf.getValue();
- Object o;
- try {
- o = objectMapper.readValue( serilizedObj, bf.getClassinfo() );
- }
- catch ( IOException e ) {
- throw new RuntimeException( "Can't deserialize object ",e );
- }
- entityMap.put( bf.getName(), o );
- }
- else {
- entityMap.put( field.getName(), field.getValue());
- }
- }
-
- return entityMap;
- }
-
-
- private static Collection processCollectionForMap(Collection c) {
- if (c.isEmpty()) {
- return c;
- }
- List processed = new ArrayList();
- Object sample = c.iterator().next();
-
- if (sample instanceof Entity) {
- for (Object o : c.toArray()) {
- Entity e = (Entity) o;
- processed.add(toMap(e));
- }
-
- } else if (sample instanceof List) {
- for (Object o : c.toArray()) {
- List list = (List) o;
- processed.add(processCollectionForMap(list)); // recursion;
- }
-
- } else if (sample instanceof Set) {
- for (Object o : c.toArray()) {
- Set set = (Set) o;
- processed.add(processCollectionForMap(set)); // recursion;
- }
-
- } else {
- for (Object o : c.toArray()) {
- processed.add(o);
- }
- }
- return processed;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
index d10025c..62ea81f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpManagerCache.java
@@ -28,7 +28,7 @@ import org.apache.usergrid.persistence.map.MapManagerFactory;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.utils.LRUCache2;
-class CpManagerCache {
+public class CpManagerCache {
private final EntityCollectionManagerFactory ecmf;
private final EntityIndexFactory eif;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
deleted file mode 100644
index 45c39ab..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpNamingUtils.java
+++ /dev/null
@@ -1,106 +0,0 @@
-package org.apache.usergrid.corepersistence;/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.usergrid.persistence.Schema;
-
-
-/**
- * Utilises for constructing standard naming conventions for collections and connections
- */
-public class CpNamingUtils {
-
- /**
- * Edge types for all types
- */
- static final String ALL_TYPES = "zzzalltypeszzz";
-
- /**
- * Edge types for collection suffix
- */
- static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
-
- /**
- * Edge types for connection suffix
- */
- static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
-
-
- static String getCollectionScopeNameFromEntityType( String type ) {
- String csn = EDGE_COLL_SUFFIX + Schema.defaultCollectionName( type );
- return csn.toLowerCase();
- }
-
-
- static String getCollectionScopeNameFromCollectionName( String name ) {
- String csn = EDGE_COLL_SUFFIX + name;
- return csn.toLowerCase();
- }
-
-
- static String getConnectionScopeName( String entityType, String connectionType ) {
- String csn = EDGE_CONN_SUFFIX + connectionType + entityType;
- return csn.toLowerCase();
- }
-
-
- static boolean isCollectionEdgeType( String type ) {
- return type.startsWith( EDGE_COLL_SUFFIX );
- }
-
-
- static boolean isConnectionEdgeType( String type ) {
- return type.startsWith( EDGE_CONN_SUFFIX );
- }
-
-
- static public String getConnectionType( String edgeType ) {
- String[] parts = edgeType.split( "\\|" );
- return parts[1];
- }
-
-
- static public String getCollectionName( String edgeType ) {
- String[] parts = edgeType.split( "\\|" );
- return parts[1];
- }
-
-
- static String getEdgeTypeFromConnectionType( String connectionType ) {
-
- if ( connectionType != null ) {
- String csn = EDGE_CONN_SUFFIX + "|" + connectionType;
- return csn;
- }
-
- return null;
- }
-
-
- static String getEdgeTypeFromCollectionName( String collectionName ) {
-
- if ( collectionName != null ) {
- String csn = EDGE_COLL_SUFFIX + "|" + collectionName;
- return csn;
- }
-
-
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 03aada3..0f31c83 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -32,6 +32,10 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.corepersistence.results.ResultsLoaderFactory;
+import org.apache.usergrid.corepersistence.results.ResultsLoaderFactoryImpl;
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
@@ -175,6 +179,8 @@ public class CpRelationManager implements RelationManager {
private IndexBucketLocator indexBucketLocator;
+ private ResultsLoaderFactory resultsLoaderFactory;
+
public CpRelationManager() {}
@@ -230,6 +236,8 @@ public class CpRelationManager implements RelationManager {
// commented out because it is possible that CP entity has not been created yet
Assert.notNull( cpHeadEntity, "cpHeadEntity cannot be null" );
+ this.resultsLoaderFactory = new ResultsLoaderFactoryImpl( managerCache );
+
return this;
}
@@ -625,7 +633,7 @@ public class CpRelationManager implements RelationManager {
memberScope.getApplication(),
memberScope.getOwner(),
memberScope.getName(),
- CpEntityMapUtils.toMap(memberEntity)
+ CpEntityMapUtils.toMap( memberEntity )
});
}
@@ -1555,136 +1563,8 @@ public class CpRelationManager implements RelationManager {
logger.debug("buildResults() for {} from {} candidates", collName, crs.size());
- //TODO T.N Change to results loader here
-
- Results results = null;
-
- EntityIndex index = managerCache.getEntityIndex(applicationScope);
- EntityIndexBatch indexBatch = index.createBatch();
-
-
- // map of the latest versions, we will discard stale indexes
- Map<Id, CandidateResult> latestVersions = new LinkedHashMap<Id, CandidateResult>();
-
- Iterator<CandidateResult> iter = crs.iterator();
- while ( iter.hasNext() ) {
-
- CandidateResult cr = iter.next();
-
- CollectionScope collScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
-
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
-
- UUID latestVersion = ecm.getLatestVersion( Collections.singleton( cr.getId()) ).toBlocking().last().getMaxVersion( cr.getId() ).getVersion();
-
- if ( logger.isDebugEnabled() ) {
- logger.debug("Getting version for entity {} from scope\n app {}\n owner {}\n name {}",
- new Object[] {
- cr.getId(),
- collScope.getApplication(),
- collScope.getOwner(),
- collScope.getName()
- });
- }
-
- if ( latestVersion == null ) {
- logger.error("Version for Entity {}:{} not found",
- cr.getId().getType(), cr.getId().getUuid());
- continue;
- }
-
- if ( cr.getVersion().compareTo( latestVersion) < 0 ) {
- logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] { cr.getId().getUuid(), cr.getId().getType(),
- cr.getVersion(), latestVersion});
-
- IndexScope indexScope = new IndexScopeImpl(
- cpHeadEntity.getId(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
- indexBatch.deindex( indexScope, cr);
-
- continue;
- }
-
- CandidateResult alreadySeen = latestVersions.get( cr.getId() );
-
- if ( alreadySeen == null ) { // never seen it, so add to map
- latestVersions.put( cr.getId(), cr );
-
- } else {
- // we seen this id before, only add entity if we now have newer version
- if ( latestVersion.compareTo( alreadySeen.getVersion() ) > 0 ) {
-
- latestVersions.put( cr.getId(), cr);
+ final Results results = this.resultsLoaderFactory.getLoader( applicationScope, this.headEntity, query.getResultsLevel() ).getResults( crs );
- IndexScope indexScope = new IndexScopeImpl(
- cpHeadEntity.getId(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
- indexBatch.deindex( indexScope, alreadySeen);
- }
- }
- }
-
- indexBatch.execute();
-
- if (query.getLevel().equals(Level.IDS)) {
-
- List<UUID> ids = new ArrayList<UUID>();
- for ( Id id : latestVersions.keySet() ) {
- CandidateResult cr = latestVersions.get(id);
- ids.add( cr.getId().getUuid() );
- }
- results = Results.fromIdList(ids);
-
- } else if (query.getLevel().equals(Level.REFS)) {
-
- List<EntityRef> refs = new ArrayList<EntityRef>();
- for ( Id id : latestVersions.keySet() ) {
- CandidateResult cr = latestVersions.get(id);
- refs.add( new SimpleEntityRef( cr.getId().getType(), cr.getId().getUuid()));
- }
- results = Results.fromRefList( refs );
-
- } else {
-
- List<Entity> entities = new ArrayList<Entity>();
- for (Id id : latestVersions.keySet()) {
-
- CandidateResult cr = latestVersions.get(id);
-
- CollectionScope collScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
-
- EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
-
- org.apache.usergrid.persistence.model.entity.Entity e =
- ecm.load( cr.getId() ).toBlocking().lastOrDefault(null);
-
- if ( e == null ) {
- logger.error("Entity {}:{} not found",
- cr.getId().getType(), cr.getId().getUuid());
- continue;
- }
-
- Entity entity = EntityFactory.newEntity(
- e.getId().getUuid(), e.getField("type").getValue().toString());
-
- Map<String, Object> entityMap = CpEntityMapUtils.toMap(e);
- entity.addProperties(entityMap);
- entities.add(entity);
- }
-
- if (entities.size() == 1) {
- results = Results.fromEntity(entities.get(0));
- } else {
- results = Results.fromEntities(entities);
- }
- }
results.setCursor( crs.getCursor() );
results.setQueryProcessor( new CpQueryProcessor(em, query, headEntity, collName) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java
deleted file mode 100644
index b034a53..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.corepersistence.results;/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import java.util.Iterator;
-import java.util.LinkedHashMap;
-import java.util.Map;
-import java.util.UUID;
-
-import org.apache.usergrid.corepersistence.CpNamingUtils;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.IndexScope;
-import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
-import org.apache.usergrid.persistence.index.query.CandidateResult;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-public abstract class AbstractIdLoader implements ResultsLoader{
-
- @Override
- public Results getResults( final ApplicationScope applicationScope, final CandidateResults crs ) {
-// Map<Id, CandidateResult> latestVersions = new LinkedHashMap<Id, CandidateResult>();
-//
-// Iterator<CandidateResult> iter = crs.iterator();
-// while ( iter.hasNext() ) {
-//
-// CandidateResult cr = iter.next();
-//
-// CollectionScope collScope = new CollectionScopeImpl(
-// applicationScope.getApplication(),
-// applicationScope.getApplication(),
-// CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
-//
-// EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
-//
-// UUID latestVersion = ecm.getLatestVersion( cr.getId() ).toBlocking().lastOrDefault(null);
-//
-// if ( logger.isDebugEnabled() ) {
-// logger.debug("Getting version for entity {} from scope\n app {}\n owner {}\n name {}",
-// new Object[] {
-// cr.getId(),
-// collScope.getApplication(),
-// collScope.getOwner(),
-// collScope.getName()
-// });
-// }
-//
-// if ( latestVersion == null ) {
-// logger.error("Version for Entity {}:{} not found",
-// cr.getId().getType(), cr.getId().getUuid());
-// continue;
-// }
-//
-// if ( cr.getVersion().compareTo( latestVersion) < 0 ) {
-// logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
-// new Object[] { cr.getId().getUuid(), cr.getId().getType(),
-// cr.getVersion(), latestVersion});
-//
-// IndexScope indexScope = new IndexScopeImpl(
-// cpHeadEntity.getId(),
-// CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
-// indexBatch.deindex( indexScope, cr);
-//
-// continue;
-// }
-//
-// CandidateResult alreadySeen = latestVersions.get( cr.getId() );
-//
-// if ( alreadySeen == null ) { // never seen it, so add to map
-// latestVersions.put( cr.getId(), cr );
-//
-// } else {
-// // we seen this id before, only add entity if we now have newer version
-// if ( latestVersion.compareTo( alreadySeen.getVersion() ) > 0 ) {
-//
-// latestVersions.put( cr.getId(), cr);
-//
-// IndexScope indexScope = new IndexScopeImpl(
-// cpHeadEntity.getId(),
-// CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
-// indexBatch.deindex( indexScope, alreadySeen);
-// }
-// }
-// }
-//
-// indexBatch.execute();
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java
deleted file mode 100644
index d20f677..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.corepersistence.results;/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-
-
-public class EntitiesLoader implements ResultsLoader {
- @Override
- public Results getResults( final ApplicationScope scope, final CandidateResults crs ) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
new file mode 100644
index 0000000..40cf499
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
@@ -0,0 +1,142 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Optional;
+
+
+/**
+ * A loader that verifies versions are correct in cassandra and match elasticsearch
+ */
+public class EntityVerifier implements ResultsVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
+
+ private EntitySet ids;
+
+ private Map<Id, org.apache.usergrid.persistence.model.entity.Entity> entityMapping;
+
+
+ public EntityVerifier( final int maxSize ) {
+ this.entityMapping = new HashMap<>( maxSize );
+ }
+
+
+ @Override
+ public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
+ ids = ecm.load( idsToLoad ).toBlocking().last();
+ }
+
+
+ @Override
+ public boolean isValid( final CandidateResult candidateResult ) {
+ final Id entityId = candidateResult.getId();
+
+ final MvccEntity version = ids.getEntity( entityId );
+
+ //version wasn't found ,deindex
+ if ( version == null ) {
+ logger.warn( "Version for Entity {}:{} not found", entityId.getUuid(), entityId.getUuid() );
+
+
+ return false;
+ }
+
+ final UUID savedVersion = version.getVersion();
+
+ if ( UUIDComparator.staticCompare( savedVersion, candidateResult.getVersion() ) > 0 ) {
+ logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), savedVersion, candidateResult.getVersion()
+ } );
+
+ return false;
+ }
+
+
+ final Optional<org.apache.usergrid.persistence.model.entity.Entity> entity = version.getEntity();
+
+ if ( !entity.isPresent() ) {
+ return false;
+ }
+
+ entityMapping.put( entityId, entity.get() );
+
+ return true;
+ }
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+
+ final List<Entity> ugEntities = new ArrayList<>( ids.size() );
+
+ for ( final Id id : ids ) {
+ final org.apache.usergrid.persistence.model.entity.Entity cpEntity = entityMapping.get( id );
+
+ Entity entity = EntityFactory.newEntity( id.getUuid(), id.toString() );
+
+ Map<String, Object> entityMap = CpEntityMapUtils.toMap( cpEntity );
+ entity.addProperties( entityMap );
+ ugEntities.add( entity );
+ }
+
+ return Results.fromEntities( ugEntities );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
new file mode 100644
index 0000000..3db772b
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -0,0 +1,240 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.corepersistence.CpManagerCache;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Function;
+import com.google.common.collect.Collections2;
+import com.google.common.collect.HashMultimap;
+
+
+public class FilteringLoader implements ResultsLoader {
+
+ private static final Logger logger = LoggerFactory.getLogger( FilteringLoader.class );
+
+ private final CpManagerCache managerCache;
+ private final ResultsVerifier resultsLoader;
+ private final Id ownerId;
+ private final ApplicationScope applicationScope;
+
+
+ protected FilteringLoader( final CpManagerCache managerCache, final ResultsVerifier resultsLoader,
+ final EntityRef ownerId, final ApplicationScope applicationScope ) {
+ this.managerCache = managerCache;
+ this.resultsLoader = resultsLoader;
+ this.ownerId = new SimpleId( ownerId.getUuid(), ownerId.getType());
+ this.applicationScope = applicationScope;
+ }
+
+
+ @Override
+ public Results getResults( final CandidateResults crs ) {
+
+
+ final EntityIndex index = managerCache.getEntityIndex( applicationScope );
+
+ final EntityIndexBatch indexBatch = index.createBatch();
+
+ final Map<Id, Integer> orderIndex = new HashMap<>( crs.size() );
+
+ final Map<Id, CandidateResult> idResultMapping = new HashMap<>( crs.size() );
+
+ final HashMultimap<String, CandidateResult> groupedByScopes = HashMultimap.create( crs.size(), crs.size() );
+
+ final Iterator<CandidateResult> iter = crs.iterator();
+
+
+ /**
+ * Go through the candidates and group them by scope for more efficient retrieval
+ */
+ for ( int i = 0; iter.hasNext(); i++ ) {
+
+ final CandidateResult cr = iter.next();
+
+ final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() );
+
+ final Id entityId = cr.getId();
+
+ //if we've already seen this one, put which ever is greater
+
+ final CandidateResult seen = idResultMapping.get( entityId );
+
+ if ( seen == null ) {
+ idResultMapping.put( entityId, cr );
+ orderIndex.put( entityId, i );
+ groupedByScopes.put( collectionType, cr );
+ }
+
+ //we have seen it, compare them
+ else {
+ final UUID seenVersion = seen.getVersion();
+ final UUID currentVersion = cr.getVersion();
+
+ //this is a newer version, we know we already have a stale entity, add it to be cleaned up
+ if ( UUIDComparator.staticCompare( currentVersion, seenVersion ) > 0 ) {
+
+ //de-index it
+ logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), seen, currentVersion
+ } );
+
+ //deindex
+ deIndex( indexBatch, ownerId, cr );
+
+
+ //TODO, fire the entity repair cleanup task here instead of de-indexing
+
+ //replace the value with a more current version
+ idResultMapping.put( entityId, cr );
+ orderIndex.put( entityId, i );
+ groupedByScopes.put( collectionType, cr );
+ }
+ }
+ }
+
+
+ //now everything is ordered, and older versions are removed. Batch fetch versions to verify existence and
+ // correct versions
+
+ final TreeMap<Integer, Id> sortedResults = new TreeMap<>();
+
+ for ( final String scopeName : groupedByScopes.keys() ) {
+
+
+ final Set<CandidateResult> candidateResults = groupedByScopes.get( scopeName );
+
+ final Collection<Id> idsToLoad =
+ Collections2.transform( candidateResults, new Function<CandidateResult, Id>() {
+ @Nullable
+ @Override
+ public Id apply( @Nullable final CandidateResult input ) {
+ if ( input == null ) {
+ return null;
+ }
+
+ return input.getId();
+ }
+ } );
+
+
+ //now using the scope, load the collection
+
+ /**
+ * Get the collection scope and batch load all the versions
+ */
+ final CollectionScope collScope =
+ new CollectionScopeImpl( applicationScope.getApplication(), applicationScope.getApplication(),
+ scopeName );
+
+
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collScope );
+
+
+ //load the results into the loader for this ech
+ resultsLoader.loadResults( idsToLoad, ecm );
+
+
+
+
+
+ //now compare them
+
+ for ( final Id requestedId : idsToLoad ) {
+
+ final CandidateResult cr = idResultMapping.get( requestedId );
+
+ //ask the loader if this is valid, if not discard it
+ if(!resultsLoader.isValid( cr )){
+ deIndex( indexBatch, ownerId, cr );
+ continue;
+ }
+
+ //if we get here we're good, we need to add this to our results
+ final int candidateIndex = orderIndex.get( requestedId );
+
+ sortedResults.put( candidateIndex, requestedId );
+
+ }
+ }
+
+
+ return resultsLoader.getResults(sortedResults.values());
+ }
+
+
+
+
+
+ protected void deIndex( final EntityIndexBatch batch, final Id ownerId, final CandidateResult candidateResult ) {
+ IndexScope indexScope = new IndexScopeImpl( ownerId,
+ CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType() ) );
+
+ batch.deindex( indexScope, candidateResult );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java
deleted file mode 100644
index 7796028..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.corepersistence.results;/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-
-
-public class IdsLoader extends AbstractIdLoader{
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java
new file mode 100644
index 0000000..cf13c74
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsVerifier.java
@@ -0,0 +1,65 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class IdsVerifier extends VersionVerifier {
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+
+ final List<UUID> returnIds = new ArrayList<>( ids.size() );
+
+ for ( final Id id : ids ) {
+ returnIds.add( id.getUuid() );
+ }
+
+
+ return Results.fromIdList( returnIds );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java
deleted file mode 100644
index ad852d5..0000000
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one or more
- * * contributor license agreements. The ASF licenses this file to You
- * * under the Apache License, Version 2.0 (the "License"); you may not
- * * use this file except in compliance with the License.
- * * You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing, software
- * * distributed under the License is distributed on an "AS IS" BASIS,
- * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * * See the License for the specific language governing permissions and
- * * limitations under the License. For additional information regarding
- * * copyright in this work, please see the NOTICE file in the top level
- * * directory of this distribution.
- *
- */
-
-package org.apache.usergrid.corepersistence.results;/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.index.query.CandidateResults;
-
-
-public class RefsLoader extends AbstractIdLoader{
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java
new file mode 100644
index 0000000..77443ee
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsVerifier.java
@@ -0,0 +1,62 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class RefsVerifier extends VersionVerifier {
+
+
+ @Override
+ public Results getResults( final Collection<Id> ids ) {
+ List<EntityRef> refs = new ArrayList<EntityRef>();
+ for ( Id id : ids ) {
+ refs.add( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+ }
+ return Results.fromRefList( refs );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
index 8a17c6e..299327d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
@@ -41,6 +41,7 @@ package org.apache.usergrid.corepersistence.results;/*
import org.apache.usergrid.persistence.Results;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Id;
/**
@@ -50,8 +51,8 @@ public interface ResultsLoader {
/**
* Using the candidate results, get the results
- * @param crs
+ * @param crs The candidate result set
* @return
*/
- public Results getResults(final ApplicationScope scope, final CandidateResults crs);
+ public Results getResults( final CandidateResults crs);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
index 5c5892c..9954597 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
@@ -38,6 +38,12 @@ package org.apache.usergrid.corepersistence.results;/*
*/
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
/**
* Factory for creating results
*/
@@ -49,5 +55,6 @@ public interface ResultsLoaderFactory {
* Get the load for results
* @return
*/
- public ResultsLoader getLoader();
+ public ResultsLoader getLoader(final ApplicationScope applicationScope, final EntityRef ownerId,
+ final Query.Level resultsLevel );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
index edaca78..7839a13 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
@@ -38,33 +38,51 @@ package org.apache.usergrid.corepersistence.results;/*
*/
+import org.junit.rules.Verifier;
+
+import org.apache.usergrid.corepersistence.CpManagerCache;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Id;
+
import com.google.inject.Inject;
-import com.google.inject.Singleton;
/**
* Factory for creating results
*/
-@Singleton
+
public class ResultsLoaderFactoryImpl implements ResultsLoaderFactory {
- private final EntitiesLoader entityLoader;
- private final IdsLoader idsLoader;
- private final RefsLoader refsLoader;
+ private final CpManagerCache managerCache;
@Inject
- public ResultsLoaderFactoryImpl( final EntitiesLoader entityLoader, final IdsLoader idsLoader,
- final RefsLoader refsLoader ) {
- this.entityLoader = entityLoader;
- this.idsLoader = idsLoader;
- this.refsLoader = refsLoader;
+ public ResultsLoaderFactoryImpl( final CpManagerCache managerCache ) {
+ this.managerCache = managerCache;
}
@Override
- public ResultsLoader getLoader() {
- return null;
+ public ResultsLoader getLoader( final ApplicationScope applicationScope, final EntityRef ownerId,
+ final Query.Level resultsLevel ) {
+
+
+ ResultsVerifier verifier;
+
+ if ( resultsLevel == Query.Level.REFS ) {
+ verifier = new RefsVerifier();
+ }
+ else if ( resultsLevel == Query.Level.IDS ) {
+ verifier = new RefsVerifier();
+ }
+ else {
+ verifier = new EntityVerifier(Query.MAX_LIMIT);
+ }
+
+
+ return new FilteringLoader( managerCache, verifier, ownerId, applicationScope );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java
new file mode 100644
index 0000000..1180ece
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsVerifier.java
@@ -0,0 +1,72 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.Collection;
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public interface ResultsVerifier {
+
+ /**
+ * Load all the candidate ides for verification
+ * @param ids The Id's to load
+ * @param ecm The entity collection manager
+ */
+ public void loadResults(Collection<Id> ids, EntityCollectionManager ecm);
+
+ /**
+ * Return true if the candidate result is a valid result that should be retained. If it should not
+ * it should also be removed from the list of possible return values in this loader
+ * @param candidateResult
+ * @return
+ */
+ public boolean isValid(CandidateResult candidateResult);
+
+
+ /**
+ * Load the result set with the given ids
+ * @return
+ */
+ public Results getResults(Collection<Id> ids);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6285e605/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java
new file mode 100644
index 0000000..778b0c3
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/VersionVerifier.java
@@ -0,0 +1,103 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.Collection;
+import java.util.UUID;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+
+
+/**
+ * A loader that verifies versions are correct in cassandra and match elasticsearch
+ */
+public abstract class VersionVerifier implements ResultsVerifier {
+
+ private static final Logger logger = LoggerFactory.getLogger( VersionVerifier.class );
+
+ private VersionSet ids;
+
+
+ @Override
+ public void loadResults( final Collection<Id> idsToLoad, final EntityCollectionManager ecm ) {
+ ids = ecm.getLatestVersion( idsToLoad ).toBlocking().last();
+ }
+
+
+ @Override
+ public boolean isValid( final CandidateResult candidateResult ) {
+ final Id entityId = candidateResult.getId();
+
+ final MvccLogEntry version = ids.getMaxVersion( entityId );
+
+ //version wasn't found ,deindex
+ if ( version == null ) {
+ logger.warn( "Version for Entity {}:{} not found", entityId.getUuid(), entityId.getUuid() );
+
+
+ return false;
+ }
+
+ final UUID savedVersion = version.getVersion();
+
+ if ( UUIDComparator.staticCompare( savedVersion, candidateResult.getVersion() ) > 0 ) {
+ logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), savedVersion, candidateResult.getVersion()
+ } );
+
+ return false;
+ }
+
+
+ return true;
+ }
+
+
+}
[04/16] git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
collection_multiget
Posted by to...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into collection_multiget
Conflicts:
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ac61f132
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ac61f132
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ac61f132
Branch: refs/heads/two-dot-o
Commit: ac61f132372f653cf14561fe7a03cb7cb6d8a79c
Parents: f1a88aa 81d4e0e
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 11:47:51 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 11:47:51 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 179 +++++++++++--------
.../corepersistence/StaleIndexCleanupTest.java | 43 ++++-
.../collection/EntityCollectionManager.java | 5 +
.../exception/WriteUniqueVerifyException.java | 1 -
.../collection/guice/CollectionModule.java | 12 +-
.../impl/EntityCollectionManagerImpl.java | 13 +-
.../collection/EntityCollectionManagerIT.java | 17 ++
7 files changed, 175 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ac61f132/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index e5917b0,6756cdc..2625078
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@@ -19,8 -19,7 +19,9 @@@
package org.apache.usergrid.persistence.collection;
+import java.util.Collection;
+
+ import java.util.UUID;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@@ -53,17 -52,16 +54,21 @@@ public interface EntityCollectionManage
*/
public Observable<Entity> load( Id entityId );
+ /**
+ * Return the latest version of the specified entity.
+ */
+ public Observable<UUID> getLatestVersion( Id entityId );
+ //TODO TN Change load to use multiget and return multiple entities. Only supports loading 1k per load operation.
+
+ /**
+ * Load all the entityIds into the observable entity set
+ * @param entityIds
+ * @return
+ */
+ public Observable<EntitySet> load(Collection<Id> entityIds);
- //TODO TN Change load to use multiget and return multiple entities.
- // Only supports loading 1k per load operation.
- //TODO Dave add a load versions using a multiget that will return a latest version
- //structure for a collection of entity Ids
+ //TODO Dave add a load versions using a multiget that will return a latest version structure for a collection of entity Ids
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ac61f132/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 39df9cb,0efb21c..5fe2fbb
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -80,7 -76,11 +83,8 @@@ public class EntityCollectionManagerImp
private final WriteOptimisticVerify writeOptimisticVerify;
private final WriteCommit writeCommit;
private final RollbackAction rollback;
+ private final GetVersion getVersion;
- //load stages
- private final Load load;
-
//delete stages
private final MarkStart markStart;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ac61f132/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index 823cb87,34061ee..a7f9ea2
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@@ -402,130 -389,24 +402,147 @@@ public class EntityCollectionManagerIT
assertNotNull( "A new version must be assigned", newVersion );
// new Version should be > old version
- assertTrue( UUIDComparator.staticCompare( newVersion, oldVersion ) > 0);
+ assertTrue( UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
+ }
+
+
+ @Test
+ public void writeMultiget() {
+
+ final CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final int multigetSize = serializationFig.getMaxLoadSize();
+
+ final List<Entity> writtenEntities = new ArrayList<>( multigetSize );
+ final List<Id> entityIds = new ArrayList<>( multigetSize );
+
+ for ( int i = 0; i < multigetSize; i++ ) {
+ final Entity entity = new Entity( new SimpleId( "test" ) );
+
+ final Entity written = manager.write( entity ).toBlocking().last();
+
+ writtenEntities.add( written );
+ entityIds.add( written.getId() );
+ }
+
+
+ final EntitySet entitySet = manager.load( entityIds ).toBlocking().lastOrDefault( null );
+
+ assertNotNull( entitySet );
+
+ assertEquals(multigetSize, entitySet.size());
+ assertFalse(entitySet.isEmpty());
+
+ /**
+ * Validate every element exists
+ */
+ for(int i = 0; i < multigetSize; i ++){
+ final Entity expected = writtenEntities.get( i );
+
+ final MvccEntity returned = entitySet.getEntity( expected.getId() );
+
+ assertEquals("Same entity returned", expected, returned.getEntity().get());
+ }
+
+
}
+
+ /**
+ * Perform a multiget where every entity will need repaired on load
+ */
+ @Test
+ public void writeMultigetRepair() {
+
+ final CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final int multigetSize = serializationFig.getMaxLoadSize();
+
+ final List<Entity> writtenEntities = new ArrayList<>( multigetSize );
+ final List<Id> entityIds = new ArrayList<>( multigetSize );
+
+ for ( int i = 0; i < multigetSize; i++ ) {
+ final Entity entity = new Entity( new SimpleId( "test" ) );
+
+ final Entity written = manager.write( entity ).toBlocking().last();
+
+ written.setField( new BooleanField( "updated", true ) );
+
+ final Entity updated = manager.update( written ).toBlocking().last();
+
+ writtenEntities.add( updated );
+ entityIds.add( updated.getId() );
+ }
+
+
+ final EntitySet entitySet = manager.load( entityIds ).toBlocking().lastOrDefault( null );
+
+ assertNotNull( entitySet );
+
+ assertEquals(multigetSize, entitySet.size());
+ assertFalse(entitySet.isEmpty());
+
+ /**
+ * Validate every element exists
+ */
+ for(int i = 0; i < multigetSize; i ++){
+ final Entity expected = writtenEntities.get( i );
+
+ final MvccEntity returned = entitySet.getEntity( expected.getId() );
+
+ assertEquals("Same entity returned", expected, returned.getEntity().get());
+
+ assertTrue( ( Boolean ) returned.getEntity().get().getField( "updated" ).getValue() );
+ }
+
+
+ }
+
+
+
+ @Test(expected = IllegalArgumentException.class)
+ public void readTooLarge() {
+
+ final CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final int multigetSize = serializationFig.getMaxLoadSize() +1;
+
+
+ final List<Id> entityIds = new ArrayList<>( multigetSize );
+
+ for ( int i = 0; i < multigetSize; i++ ) {
+
+ entityIds.add( new SimpleId( "simple" ) );
+ }
+
+
+ //should throw an exception
+ manager.load( entityIds ).toBlocking().lastOrDefault( null );
+
+
+
+ }
+ @Test
+ public void testGetVersion() {
+
+ CollectionScope context = new CollectionScopeImpl(
+ new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+
+ Entity newEntity = new Entity( new SimpleId( "test" ) );
+ EntityCollectionManager manager = factory.createCollectionManager( context );
+ Observable<Entity> observable = manager.write( newEntity );
+ Entity created = observable.toBlocking().lastOrDefault( null );
+
+ assertNotNull("Id was assigned", created.getId() );
+ assertNotNull("Version was assigned", created.getVersion() );
+
+ assertTrue(UUIDComparator.staticCompare(created.getVersion(),
+ manager.getLatestVersion( created.getId() ).toBlocking().lastOrDefault(null)) == 0);
+ }
}
[06/16] Refactor of loader started
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
index 8bd98f4..e4aeb0b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImpl.java
@@ -20,8 +20,10 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -35,8 +37,10 @@ import org.apache.cassandra.db.marshal.ReversedType;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
@@ -54,9 +58,9 @@ import com.netflix.astyanax.ColumnListMutation;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ColumnList;
+import com.netflix.astyanax.model.Row;
import com.netflix.astyanax.serializers.AbstractSerializer;
import com.netflix.astyanax.serializers.UUIDSerializer;
@@ -69,13 +73,13 @@ import com.netflix.astyanax.serializers.UUIDSerializer;
@Singleton
public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerializationStrategy, Migration {
- private static final Logger LOG = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
+ private static final Logger LOG = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
private static final StageSerializer SER = new StageSerializer();
private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
- private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
+ private static final CollectionScopedRowKeySerializer<Id> ROW_KEY_SER =
new CollectionScopedRowKeySerializer<Id>( ID_SER );
private static final MultiTennantColumnFamily<CollectionScope, Id, UUID> CF_ENTITY_LOG =
@@ -102,7 +106,7 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
final Stage stage = entry.getStage();
final UUID colName = entry.getVersion();
- final StageStatus stageStatus = new StageStatus(stage,entry.getState());
+ final StageStatus stageStatus = new StageStatus( stage, entry.getState() );
return doWrite( collectionScope, entry.getEntityId(), entry.getVersion(), new RowOp() {
@Override
@@ -122,44 +126,90 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
@Override
- public MvccLogEntry load( final CollectionScope collectionScope, final Id entityId, final UUID version )
- throws ConnectionException {
+ public VersionSet load( final CollectionScope collectionScope, final Collection<Id> entityIds,
+ final UUID maxVersion ) {
Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
- Preconditions.checkNotNull( entityId, "entity id is required" );
- Preconditions.checkNotNull( version, "version is required" );
+ Preconditions.checkNotNull( entityIds, "entityIds is required" );
+ Preconditions.checkArgument( entityIds.size() > 0, "You must specify an Id" );
+ Preconditions.checkNotNull( maxVersion, "maxVersion is required" );
+
+
+ //didnt put the max in the error message, I don't want to take the string construction hit every time
+ Preconditions.checkArgument( entityIds.size() <= fig.getMaxLoadSize(),
+ "requested size cannot be over configured maximum" );
+
+
+ final List<ScopedRowKey<CollectionScope, Id>> rowKeys = new ArrayList<>( entityIds.size() );
+
+
+ for ( final Id entityId : entityIds ) {
+ rowKeys.add( ScopedRowKey.fromKey( collectionScope, entityId ) );
+ }
- Column<UUID> result;
+ final Iterator<Row<ScopedRowKey<CollectionScope, Id>, UUID>> latestEntityColumns;
+
try {
- result = keyspace.prepareQuery( CF_ENTITY_LOG ).getKey( ScopedRowKey
- .fromKey( collectionScope, entityId ) )
- .getColumn( version ).execute().getResult();
+ latestEntityColumns = keyspace.prepareQuery( CF_ENTITY_LOG ).getKeySlice( rowKeys )
+ .withColumnRange( maxVersion, null, false, 1 ).execute().getResult()
+ .iterator();
}
- catch ( NotFoundException nfe ) {
- return null;
+ catch ( ConnectionException e ) {
+ throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra",
+ e );
}
- final StageStatus stageStatus = result.getValue( SER );
+ final VersionSetImpl versionResults = new VersionSetImpl( entityIds.size() );
+
+ while ( latestEntityColumns.hasNext() ) {
+ final Row<ScopedRowKey<CollectionScope, Id>, UUID> row = latestEntityColumns.next();
+
+ final ColumnList<UUID> columns = row.getColumns();
+
+ if ( columns.size() == 0 ) {
+ continue;
+ }
+
+
+ final Id entityId = row.getKey().getKey();
+
+ final Column<UUID> column = columns.getColumnByIndex( 0 );
- return new MvccLogEntryImpl( entityId, version, stageStatus.stage, stageStatus.state );
+
+ final UUID version = column.getName();
+
+ final StageStatus stageStatus = column.getValue( SER );
+
+ final MvccLogEntry logEntry =
+ new MvccLogEntryImpl( entityId, version, stageStatus.stage, stageStatus.state );
+
+
+ versionResults.addEntry( logEntry );
+ }
+
+ return versionResults;
}
@Override
public List<MvccLogEntry> load( final CollectionScope collectionScope, final Id entityId, final UUID version,
- final int maxSize ) throws ConnectionException {
+ final int maxSize ) {
Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
Preconditions.checkNotNull( entityId, "entity id is required" );
Preconditions.checkNotNull( version, "version is required" );
Preconditions.checkArgument( maxSize > 0, "max Size must be greater than 0" );
- ColumnList<UUID> columns =
- keyspace.prepareQuery( CF_ENTITY_LOG ).getKey( ScopedRowKey
- .fromKey( collectionScope, entityId ) )
- .withColumnRange( version, null, false, maxSize ).execute().getResult();
+ ColumnList<UUID> columns = null;
+ try {
+ columns = keyspace.prepareQuery( CF_ENTITY_LOG ).getKey( ScopedRowKey.fromKey( collectionScope, entityId ) )
+ .withColumnRange( version, null, false, maxSize ).execute().getResult();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( "Unable to load log entries", e );
+ }
List<MvccLogEntry> results = new ArrayList<MvccLogEntry>( columns.size() );
@@ -182,7 +232,7 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
Preconditions.checkNotNull( entityId, "entityId is required" );
Preconditions.checkNotNull( version, "version context is required" );
- return doWrite( context, entityId, version, new RowOp() {
+ return doWrite( context, entityId, version, new RowOp() {
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
colMutation.deleteColumn( version );
@@ -192,13 +242,13 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
@Override
- public java.util.Collection getColumnFamilies() {
+ public Collection<MultiTennantColumnFamilyDefinition> getColumnFamilies() {
//create the CF entity data. We want it reversed b/c we want the most recent version at the top of the
//row for fast seeks
- MultiTennantColumnFamilyDefinition cf = new MultiTennantColumnFamilyDefinition( CF_ENTITY_LOG,
- BytesType.class.getSimpleName(),
- ReversedType.class.getSimpleName() + "(" + UUIDType.class.getSimpleName() + ")",
- IntegerType.class.getSimpleName() , MultiTennantColumnFamilyDefinition.CacheOption.KEYS);
+ MultiTennantColumnFamilyDefinition cf =
+ new MultiTennantColumnFamilyDefinition( CF_ENTITY_LOG, BytesType.class.getSimpleName(),
+ ReversedType.class.getSimpleName() + "(" + UUIDType.class.getSimpleName() + ")",
+ IntegerType.class.getSimpleName(), MultiTennantColumnFamilyDefinition.CacheOption.KEYS );
return Collections.singleton( cf );
@@ -261,12 +311,13 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
}
}
+
/**
* Internal stage shard
*/
private static class StatusCache {
- private Map<Integer, MvccLogEntry.State> values = new HashMap<Integer, MvccLogEntry.State>( MvccLogEntry.State
- .values().length );
+ private Map<Integer, MvccLogEntry.State> values =
+ new HashMap<Integer, MvccLogEntry.State>( MvccLogEntry.State.values().length );
private StatusCache() {
@@ -287,6 +338,7 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
}
}
+
public static class StageSerializer extends AbstractSerializer<StageStatus> {
/**
@@ -295,12 +347,13 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
private static final StageCache CACHE = new StageCache();
private static final StatusCache STATUS_CACHE = new StatusCache();
+
@Override
public ByteBuffer toByteBuffer( final StageStatus obj ) {
- ByteBuffer byteBuffer = ByteBuffer.allocate(8);
- byteBuffer.putInt(obj.stage.getId());
- byteBuffer.putInt(obj.state.getId());
+ ByteBuffer byteBuffer = ByteBuffer.allocate( 8 );
+ byteBuffer.putInt( obj.stage.getId() );
+ byteBuffer.putInt( obj.state.getId() );
byteBuffer.rewind();
return byteBuffer;
}
@@ -311,18 +364,20 @@ public class MvccLogEntrySerializationStrategyImpl implements MvccLogEntrySerial
int value = byteBuffer.getInt();
Stage stage = CACHE.getStage( value );
value = byteBuffer.getInt();
- MvccLogEntry.State state = STATUS_CACHE.getStatus(value);
- return new StageStatus(stage, state );
+ MvccLogEntry.State state = STATUS_CACHE.getStatus( value );
+ return new StageStatus( stage, state );
}
}
+
public static class StageStatus {
final Stage stage;
final MvccLogEntry.State state;
- public StageStatus(Stage stage, MvccLogEntry.State state ){
+
+
+ public StageStatus( Stage stage, MvccLogEntry.State state ) {
this.stage = stage;
this.state = state;
}
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/VersionSetImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/VersionSetImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/VersionSetImpl.java
new file mode 100644
index 0000000..0adb921
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/VersionSetImpl.java
@@ -0,0 +1,80 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.collection.serialization.impl;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.VersionSet;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Implementation of the version set
+ */
+public class VersionSetImpl implements VersionSet {
+
+ private final Map<Id, MvccLogEntry> logEntries;
+
+ public VersionSetImpl(final int maxSize){
+ logEntries = new HashMap<>( maxSize );
+ }
+
+ public void addEntry(final MvccLogEntry entry){
+ this.logEntries.put( entry.getEntityId(), entry );
+ }
+
+ @Override
+ public MvccLogEntry getMaxVersion( final Id entityId ) {
+ return this.logEntries.get( entityId );
+ }
+
+
+ @Override
+ public int size() {
+ return this.logEntries.size();
+ }
+
+
+ @Override
+ public boolean isEmpty() {
+ return this.logEntries.isEmpty();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
index a7f9ea2..c01e022 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/EntityCollectionManagerIT.java
@@ -19,6 +19,7 @@ package org.apache.usergrid.persistence.collection;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.UUID;
@@ -31,7 +32,7 @@ import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyExc
import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.model.entity.Entity;
@@ -432,21 +433,19 @@ public class EntityCollectionManagerIT {
assertNotNull( entitySet );
- assertEquals(multigetSize, entitySet.size());
- assertFalse(entitySet.isEmpty());
+ assertEquals( multigetSize, entitySet.size() );
+ assertFalse( entitySet.isEmpty() );
/**
* Validate every element exists
*/
- for(int i = 0; i < multigetSize; i ++){
+ for ( int i = 0; i < multigetSize; i++ ) {
final Entity expected = writtenEntities.get( i );
final MvccEntity returned = entitySet.getEntity( expected.getId() );
- assertEquals("Same entity returned", expected, returned.getEntity().get());
+ assertEquals( "Same entity returned", expected, returned.getEntity().get() );
}
-
-
}
@@ -454,64 +453,61 @@ public class EntityCollectionManagerIT {
* Perform a multiget where every entity will need repaired on load
*/
@Test
- public void writeMultigetRepair() {
-
- final CollectionScope context =
- new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
- final EntityCollectionManager manager = factory.createCollectionManager( context );
-
- final int multigetSize = serializationFig.getMaxLoadSize();
-
- final List<Entity> writtenEntities = new ArrayList<>( multigetSize );
- final List<Id> entityIds = new ArrayList<>( multigetSize );
+ public void writeMultigetRepair() {
- for ( int i = 0; i < multigetSize; i++ ) {
- final Entity entity = new Entity( new SimpleId( "test" ) );
+ final CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
- final Entity written = manager.write( entity ).toBlocking().last();
+ final int multigetSize = serializationFig.getMaxLoadSize();
- written.setField( new BooleanField( "updated", true ) );
+ final List<Entity> writtenEntities = new ArrayList<>( multigetSize );
+ final List<Id> entityIds = new ArrayList<>( multigetSize );
- final Entity updated = manager.update( written ).toBlocking().last();
+ for ( int i = 0; i < multigetSize; i++ ) {
+ final Entity entity = new Entity( new SimpleId( "test" ) );
- writtenEntities.add( updated );
- entityIds.add( updated.getId() );
- }
+ final Entity written = manager.write( entity ).toBlocking().last();
+ written.setField( new BooleanField( "updated", true ) );
- final EntitySet entitySet = manager.load( entityIds ).toBlocking().lastOrDefault( null );
+ final Entity updated = manager.update( written ).toBlocking().last();
- assertNotNull( entitySet );
+ writtenEntities.add( updated );
+ entityIds.add( updated.getId() );
+ }
- assertEquals(multigetSize, entitySet.size());
- assertFalse(entitySet.isEmpty());
- /**
- * Validate every element exists
- */
- for(int i = 0; i < multigetSize; i ++){
- final Entity expected = writtenEntities.get( i );
+ final EntitySet entitySet = manager.load( entityIds ).toBlocking().lastOrDefault( null );
- final MvccEntity returned = entitySet.getEntity( expected.getId() );
+ assertNotNull( entitySet );
- assertEquals("Same entity returned", expected, returned.getEntity().get());
+ assertEquals( multigetSize, entitySet.size() );
+ assertFalse( entitySet.isEmpty() );
- assertTrue( ( Boolean ) returned.getEntity().get().getField( "updated" ).getValue() );
- }
+ /**
+ * Validate every element exists
+ */
+ for ( int i = 0; i < multigetSize; i++ ) {
+ final Entity expected = writtenEntities.get( i );
+ final MvccEntity returned = entitySet.getEntity( expected.getId() );
- }
+ assertEquals( "Same entity returned", expected, returned.getEntity().get() );
+ assertTrue( ( Boolean ) returned.getEntity().get().getField( "updated" ).getValue() );
+ }
+ }
- @Test(expected = IllegalArgumentException.class)
+ @Test( expected = IllegalArgumentException.class )
public void readTooLarge() {
final CollectionScope context =
new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
final EntityCollectionManager manager = factory.createCollectionManager( context );
- final int multigetSize = serializationFig.getMaxLoadSize() +1;
+ final int multigetSize = serializationFig.getMaxLoadSize() + 1;
final List<Id> entityIds = new ArrayList<>( multigetSize );
@@ -524,25 +520,47 @@ public class EntityCollectionManagerIT {
//should throw an exception
manager.load( entityIds ).toBlocking().lastOrDefault( null );
+ }
-
- }
@Test
public void testGetVersion() {
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ CollectionScope context =
+ new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+
+
+ final EntityCollectionManager manager = factory.createCollectionManager( context );
+
+ final Entity newEntity = new Entity( new SimpleId( "test" ) );
+
+ Entity created1 = manager.write( newEntity ).toBlocking().lastOrDefault( null );
+
+ assertNotNull( "Id was assigned", created1.getId() );
+ assertNotNull( "Version was assigned", created1.getVersion() );
+
+ Entity secondEntity = new Entity( new SimpleId( "test" ) );
+
+ Entity created2 = manager.write( secondEntity ).toBlocking().lastOrDefault( null );
+
+ assertNotNull( "Id was assigned", created2.getId() );
+ assertNotNull( "Version was assigned", created2.getVersion() );
+
+
+ VersionSet results =
+ manager.getLatestVersion( Arrays.asList( created1.getId(), created2.getId() ) ).toBlocking().last();
- Entity newEntity = new Entity( new SimpleId( "test" ) );
- EntityCollectionManager manager = factory.createCollectionManager( context );
- Observable<Entity> observable = manager.write( newEntity );
- Entity created = observable.toBlocking().lastOrDefault( null );
- assertNotNull("Id was assigned", created.getId() );
- assertNotNull("Version was assigned", created.getVersion() );
+ final MvccLogEntry version1Log = results.getMaxVersion( created1.getId() );
+ assertEquals( created1.getId(), version1Log.getEntityId() );
+ assertEquals( created1.getVersion(), version1Log.getVersion() );
+ assertEquals( MvccLogEntry.State.COMPLETE, version1Log.getState() );
+ assertEquals( Stage.COMMITTED, version1Log.getStage() );
- assertTrue(UUIDComparator.staticCompare(created.getVersion(),
- manager.getLatestVersion( created.getId() ).toBlocking().lastOrDefault(null)) == 0);
+ final MvccLogEntry version2Log = results.getMaxVersion( created2.getId() );
+ assertEquals( created2.getId(), version2Log.getEntityId() );
+ assertEquals( created2.getVersion(), version2Log.getVersion() );
+ assertEquals( MvccLogEntry.State.COMPLETE, version2Log.getState() );
+ assertEquals( Stage.COMMITTED, version2Log.getStage() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
index 6795893..0ca0649 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
@@ -20,14 +20,13 @@ package org.apache.usergrid.persistence.collection.mvcc.changelog;
import java.util.Arrays;
import java.util.Collection;
-import java.util.Iterator;
import java.util.Set;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImplTest.java
index 33c3102..52cf954 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImplTest.java
@@ -23,7 +23,7 @@ import java.util.UUID;
import org.junit.Test;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImplTest.java
index eb9bd18..a892238 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImplTest.java
@@ -5,7 +5,7 @@ import java.util.UUID;
import org.junit.Test;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/AbstractMvccEntityStageTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/AbstractMvccEntityStageTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/AbstractMvccEntityStageTest.java
index 950130a..6c6ea0f 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/AbstractMvccEntityStageTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/AbstractMvccEntityStageTest.java
@@ -26,7 +26,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.collection.util.InvalidEntityGenerator;
import org.apache.usergrid.persistence.collection.util.InvalidIdGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/TestEntityGenerator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/TestEntityGenerator.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/TestEntityGenerator.java
index 99cd335..576c1c6 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/TestEntityGenerator.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/TestEntityGenerator.java
@@ -20,7 +20,7 @@ package org.apache.usergrid.persistence.collection.mvcc.stage;
import java.util.UUID;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
index c2876b0..f5dc24c 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommitTest.java
@@ -1,16 +1,14 @@
package org.apache.usergrid.persistence.collection.mvcc.stage.delete;
-import java.security.Key;
-
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.stage.AbstractMvccEntityStageTest;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStartTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStartTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStartTest.java
index 1b04022..6c13dd5 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStartTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStartTest.java
@@ -8,8 +8,8 @@ import org.mockito.ArgumentCaptor;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.stage.AbstractIdStageTest;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
index b0ccb4f..93cde8e 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommitTest.java
@@ -24,8 +24,8 @@ import org.mockito.ArgumentCaptor;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.stage.AbstractMvccEntityStageTest;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java
index c64fe23..85fce69 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerifyTest.java
@@ -30,8 +30,8 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.AbstractMvccEntityStageTest;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
index 3ff3e09..e32c17c 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStartTest.java
@@ -23,8 +23,8 @@ import org.mockito.ArgumentCaptor;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.stage.AbstractEntityStageTest;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
index 1dc2f32..6551716 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerifyTest.java
@@ -26,7 +26,7 @@ import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
index d1b2961..27f39dc 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/EntityRepairImplTest.java
@@ -29,7 +29,7 @@ import org.junit.Test;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.impl.EntityRepairImpl;
import org.apache.usergrid.persistence.model.entity.Entity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java
index 9ee284b..20d478d 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIteratorTest.java
@@ -10,7 +10,7 @@ import org.junit.Test;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.util.LogEntryMock;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
index 8965668..73cf848 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImplTest.java
@@ -38,7 +38,7 @@ import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
index b1ed50d..fa80ca4 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLESSTransientTest.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.Collections;
import java.util.UUID;
import org.jukito.UseModules;
@@ -35,7 +36,7 @@ import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
@@ -80,7 +81,7 @@ public class MvccLESSTransientTest {
CollectionScope context = new CollectionScopeImpl( organizationId, applicationId, name );
- final SimpleId id = new SimpleId( "test" );
+ final Id id = new SimpleId( "test" );
final UUID version = UUIDGenerator.newTimeUUID();
for ( Stage stage : Stage.values() ) {
@@ -93,7 +94,7 @@ public class MvccLESSTransientTest {
//noinspection PointlessArithmeticExpression
Thread.sleep( 1000 );
- MvccLogEntry returned = logEntryStrategy.load( context, id, version );
+ MvccLogEntry returned = logEntryStrategy.load( context, Collections.singleton(id), version ).getMaxVersion( id );
if ( stage.isTransient() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
index 25daa3b..629dd83 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccLogEntrySerializationStrategyImplTest.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.persistence.collection.serialization.impl;
+import java.util.Collections;
import java.util.List;
import java.util.UUID;
@@ -33,7 +34,7 @@ import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
@@ -74,7 +75,7 @@ public class MvccLogEntrySerializationStrategyImplTest {
CollectionScope context = new CollectionScopeImpl(organizationId, applicationId, name );
- final SimpleId id = new SimpleId( "test" );
+ final Id id = new SimpleId( "test" );
final UUID version = UUIDGenerator.newTimeUUID();
for ( Stage stage : Stage.values() ) {
@@ -83,7 +84,7 @@ public class MvccLogEntrySerializationStrategyImplTest {
//Read it back
- MvccLogEntry returned = logEntryStrategy.load( context, id, version );
+ MvccLogEntry returned = logEntryStrategy.load( context, Collections.singleton(id), version ).getMaxVersion( id );
assertNotNull( "Returned value should not be null", returned );
@@ -103,11 +104,11 @@ public class MvccLogEntrySerializationStrategyImplTest {
CollectionScope context = new CollectionScopeImpl(organizationId, applicationId, name );
- final SimpleId id = new SimpleId( "test" );
+ final Id id = new SimpleId( "test" );
final UUID version = UUIDGenerator.newTimeUUID();
- MvccLogEntry returned = logEntryStrategy.load( context, id, version );
+ MvccLogEntry returned = logEntryStrategy.load( context, Collections.singleton(id), version ).getMaxVersion( id );
assertNull( "Returned value should not exist", returned );
}
@@ -124,7 +125,7 @@ public class MvccLogEntrySerializationStrategyImplTest {
CollectionScope context = new CollectionScopeImpl(organizationId, applicationId, name );
- final SimpleId id = new SimpleId( "test" );
+ final Id id = new SimpleId( "test" );
int count = 10;
@@ -141,7 +142,7 @@ public class MvccLogEntrySerializationStrategyImplTest {
//Read it back
- MvccLogEntry returned = logEntryStrategy.load( context, id, versions[i] );
+ MvccLogEntry returned = logEntryStrategy.load( context, Collections.singleton(id), versions[i] ).getMaxVersion( id );
assertNotNull( "Returned value should not be null", returned );
@@ -210,7 +211,7 @@ public class MvccLogEntrySerializationStrategyImplTest {
@Test(expected = NullPointerException.class)
public void loadParamContext() throws ConnectionException {
- logEntryStrategy.load( null, new SimpleId( "test" ), UUIDGenerator.newTimeUUID() );
+ logEntryStrategy.load( null, Collections.<Id>emptyList(), UUIDGenerator.newTimeUUID() );
}
@@ -227,7 +228,7 @@ public class MvccLogEntrySerializationStrategyImplTest {
logEntryStrategy
.load( new CollectionScopeImpl( new SimpleId( "organization" ), new SimpleId( "test" ), "test" ),
- new SimpleId( "test" ), null );
+ Collections.<Id>singleton( new SimpleId( "test" )), null );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidMvccEntityGenerator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidMvccEntityGenerator.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidMvccEntityGenerator.java
index 48dc38c..550f02a 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidMvccEntityGenerator.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidMvccEntityGenerator.java
@@ -31,7 +31,7 @@ import org.junit.experimental.theories.PotentialAssignment;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidValueGeneratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidValueGeneratorTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidValueGeneratorTest.java
index b2616d9..adb79ab 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidValueGeneratorTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/InvalidValueGeneratorTest.java
@@ -25,7 +25,7 @@ import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
index a25bc94..57c06f5 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/util/LogEntryMock.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.persistence.collection.util;/*
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
@@ -32,14 +31,13 @@ import org.mockito.stubbing.Answer;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.fasterxml.uuid.UUIDComparator;
-import com.google.common.base.Preconditions;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import static org.mockito.Matchers.any;
[15/16] git commit: Merge branch 'collection_multiget' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into
collection_multiget
Posted by to...@apache.org.
Merge branch 'collection_multiget' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into collection_multiget
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/454ef4ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/454ef4ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/454ef4ff
Branch: refs/heads/two-dot-o
Commit: 454ef4ff1456ed378d098054fadfde07d4145a72
Parents: 6285e60 f43e4f5
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 15:50:10 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 15:50:10 2014 -0600
----------------------------------------------------------------------
.../collection/EntityCollectionManager.java | 8 +
.../impl/EntityCollectionManagerImpl.java | 193 +++++++++++--------
.../collection/EntityCollectionManagerIT.java | 31 +++
3 files changed, 147 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
[02/16] git commit: Changes to repair to only bring us up to the
current version only. This is what we need in practice.
Posted by to...@apache.org.
Changes to repair to only bring us up to the current version only. This is what we need in practice.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f7d406e3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f7d406e3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f7d406e3
Branch: refs/heads/two-dot-o
Commit: f7d406e3a0387b1cc5738428f49c0bd89670a91f
Parents: 185ecef
Author: Todd Nine <to...@apache.org>
Authored: Wed Oct 8 15:14:01 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 11:28:45 2014 -0600
----------------------------------------------------------------------
.../collection/EntityCollectionManager.java | 9 +
.../persistence/collection/EntitySet.java | 48 +++
.../impl/EntityCollectionManagerImpl.java | 10 +
.../mvcc/MvccEntitySerializationStrategy.java | 11 +
.../collection/mvcc/changelog/ChangeLog.java | 68 ++++
.../mvcc/changelog/ChangeLogEntry.java | 104 ------
.../mvcc/changelog/ChangeLogGenerator.java | 11 +-
.../mvcc/changelog/ChangeLogGeneratorImpl.java | 136 +++----
.../mvcc/changelog/ChangeLogImpl.java | 135 +++++++
.../collection/mvcc/stage/load/Load.java | 21 +-
.../collection/serialization/EntityRepair.java | 38 ++
.../serialization/impl/EntityRepairImpl.java | 146 ++++++++
.../serialization/impl/EntitySetImpl.java | 69 ++++
.../MvccEntitySerializationStrategyImpl.java | 169 ++++++---
.../persistence/collection/util/RepairUtil.java | 139 -------
.../changelog/ChangeLogGeneratorImplTest.java | 370 +++++++++++--------
.../collection/mvcc/stage/load/LoadTest.java | 30 +-
.../serialization/EntityRepairImplTest.java | 136 +++++++
...MvccEntitySerializationStrategyImplTest.java | 2 +-
19 files changed, 1121 insertions(+), 531 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index b49989f..e5917b0 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -19,6 +19,8 @@
package org.apache.usergrid.persistence.collection;
+import java.util.Collection;
+
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -53,6 +55,13 @@ public interface EntityCollectionManager {
//TODO TN Change load to use multiget and return multiple entities. Only supports loading 1k per load operation.
+ /**
+ * Load all the entityIds into the observable entity set
+ * @param entityIds
+ * @return
+ */
+ public Observable<EntitySet> load(Collection<Id> entityIds);
+
//TODO Dave add a load versions using a multiget that will return a latest version structure for a collection of entity Ids
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
new file mode 100644
index 0000000..32c168f
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
@@ -0,0 +1,48 @@
+package org.apache.usergrid.persistence.collection;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Represents a set of entities
+ */
+public interface EntitySet {
+
+ /**
+ * Get the entity from the result set
+ * @param entityId
+ * @return
+ */
+ public Entity getEntity(Id entityId);
+
+ /**
+ * Get the number of entities in this set
+ * @return
+ */
+ public int size();
+
+ /**
+ * Return true if the set is empty
+ * @return
+ */
+ public boolean isEmpty();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index b2b07e9..4a72285 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -19,11 +19,14 @@
package org.apache.usergrid.persistence.collection.impl;
+import java.util.Collection;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
@@ -184,6 +187,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
.map( load );
}
+
+ @Override
+ public Observable<EntitySet> load( final Collection<Id> entityIds ) {
+ return null;
+ }
+
+
@Override
public Observable<Entity> update( final Entity entity ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
index b2deb1a..7947029 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
@@ -19,11 +19,13 @@
package org.apache.usergrid.persistence.collection.mvcc;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -57,6 +59,15 @@ public interface MvccEntitySerializationStrategy {
*/
public MvccEntity load( CollectionScope context, Id entityId, UUID version );
+
+ /**
+ * Load the entities into the entitySet from the specified Ids. Loads versions <= the maxVersion
+ * @param scope
+ * @param entityIds
+ * @return
+ */
+ public EntitySet load( CollectionScope scope, Collection<Id> entityIds, UUID maxVersion);
+
/**
* Load a list, from highest to lowest of the entity with versions <= version up to maxSize elements
*
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLog.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLog.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLog.java
new file mode 100644
index 0000000..d028aa0
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLog.java
@@ -0,0 +1,68 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.collection.mvcc.changelog;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.model.field.Field;
+
+
+public interface ChangeLog {
+
+
+ /**
+ * Get all fields that should be removed from the max version
+ * @return All entries that are deletes
+ */
+ public Set<String> getDeletes();
+
+ /**
+ * Get all writes to apply to the current version
+ * @return
+ */
+ public java.util.Collection<Field> getWrites();
+
+ /**
+ * Get the number of changelog entries (writes+deletes)
+ * @return
+ */
+ public int getSize();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogEntry.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogEntry.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogEntry.java
deleted file mode 100644
index e4b2c2b..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogEntry.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License. For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-package org.apache.usergrid.persistence.collection.mvcc.changelog;
-
-
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.model.entity.Id;
-import org.apache.usergrid.persistence.model.field.Field;
-
-import com.fasterxml.uuid.UUIDComparator;
-
-
-/**
- * Records one change to an entry field: entry ID, version, change type and the changed field.
- */
-public class ChangeLogEntry implements Comparable {
-
- private final Id entityId;
-
- private final Set<UUID> versions = new TreeSet<UUID>();
-
- public enum ChangeType {
- PROPERTY_WRITE,
- PROPERTY_DELETE
- };
-
- private final ChangeType changeType;
-
- private final Field field;
-
- public ChangeLogEntry(Id entryId, UUID version, ChangeType changeType, Field changedField) {
- this.entityId = entryId;
- if (version != null) {
- this.versions.add(version);
- }
- this.changeType = changeType;
- this.field = changedField;
- }
-
- public int compareTo( Object o ) {
- ChangeLogEntry other = (ChangeLogEntry)o;
- return UUIDComparator.staticCompare(entityId.getUuid(), other.entityId.getUuid());
- }
-
- /**
- * @return the entityId
- */
- public Id getEntryId() {
- return entityId;
- }
-
- /**
- * @return the version
- */
- public Set<UUID> getVersions() {
- return versions;
- }
-
- /**
- * @param version the version to set
- */
- public void addVersion( UUID version ) {
- this.versions.add(version);
- }
-
- /**
- * @return the changeType
- */
- public ChangeType getChangeType() {
- return changeType;
- }
-
- /**
- * @return the field
- */
- public Field getField() {
- return field;
- }
-
- public String toString() {
- return "Type = " + changeType.toString()
- + ", Property = " + field.getName()
- + ", Value = " + field.getValue()
- + ", Versions = " + versions.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
index 83ff95c..68cbd58 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
@@ -18,6 +18,7 @@
package org.apache.usergrid.persistence.collection.mvcc.changelog;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.UUID;
@@ -38,12 +39,8 @@ public interface ChangeLogGenerator {
*
* @param versions Versions of the entity to be considered.
* Must be ordered from lowest time UUID to highest time UUID.
- *
- * @param minVersion Properties of versions older than this should be discarded
- * and properties newer should be retained.
- *
- * @return Change-log entries ordered by version, ordered from lowest time
- * uuid to highest timeuuid.
+ *
+ * @return A ChangeLog of all changes groups by version
*/
- List<ChangeLogEntry> getChangeLog( Iterator<MvccEntity> versions, UUID minVersion );
+ public ChangeLog getChangeLog( Collection<MvccEntity> versions);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
index 0372d3f..014d249 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
@@ -18,13 +18,19 @@
package org.apache.usergrid.persistence.collection.mvcc.changelog;
-import java.util.*;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Set;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.Field;
-import com.fasterxml.uuid.UUIDComparator;
+import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
/**
@@ -33,79 +39,77 @@ import com.fasterxml.uuid.UUIDComparator;
public class ChangeLogGeneratorImpl implements ChangeLogGenerator {
/**
- * See parent comment
- * {@link ChangeLogGenerator#getChangeLog(java.util.Iterator<org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity>, java.util.UUID)}
- * @param mvccEntities
+ * See parent comment {@link ChangeLogGenerator#getChangeLog(java.util.Collection)}
*/
@Override
- public List<ChangeLogEntry> getChangeLog( Iterator<MvccEntity> mvccEntities, UUID minVersion ) {
+ public ChangeLog getChangeLog( Collection<MvccEntity> mvccEntities ) {
- Map<String, ChangeLogEntry> writeMap = new HashMap<String, ChangeLogEntry>();
- Map<String, ChangeLogEntry> deleteMap = new HashMap<String, ChangeLogEntry>();
- List<ChangeLogEntry> changeLog = new ArrayList<ChangeLogEntry>();
- Entity keeper = null;
- List<Entity> entityList = new ArrayList<>();
- while(mvccEntities.hasNext()) {
- MvccEntity mvccEntity = mvccEntities.next();
+ Preconditions.checkArgument( mvccEntities.size() > 0, "You must specify at least 1 entities for a change log" );
- Entity entity = mvccEntity.getEntity().get();
- entityList.add(entity);
- int compare = UUIDComparator.staticCompare(mvccEntity.getVersion(), minVersion);
+ //TODO, this is a SWAG on the entity size, this may be too little or too much.
+ final ChangeLogImpl changeLog = new ChangeLogImpl( 50 );
- if (compare == 0) {
- keeper = entity;
+ Iterator<MvccEntity> iterator = mvccEntities.iterator();
+
+ Set<String> previousFieldNames = getFieldNames( iterator.next().getEntity() );
+
+ Set<String> currentFieldNames = null;
+
+ while ( iterator.hasNext() ) {
+
+
+ final MvccEntity mvccEntity = iterator.next();
+
+ currentFieldNames = getFieldNames( mvccEntity.getEntity() );
+
+ if(mvccEntity.getStatus() == MvccEntity.Status.DELETED){
+ changeLog.clear();
+ continue;
}
- }
- for (Entity entity : entityList) {
-
- int compare = UUIDComparator.staticCompare(entity.getVersion(), minVersion);
-
-
- // TODO: what about cleared entities, all fields deleted but entity still there.
- // i.e. the optional entity will be delete
- if (compare < 0) { // less than minVersion
-
- for (Field field : entity.getFields()) {
-
- // only delete field if it is not in the keeper
- Field keeperField = keeper.getField(field.getName());
- if (keeperField == null
- || keeperField.getValue() == null
- || !keeperField.getValue().equals(field.getValue())) {
-
- String key = field.getName() + field.getValue();
- ChangeLogEntry cle = deleteMap.get(key);
- if (cle == null) {
- cle = new ChangeLogEntry(
- entity.getId(), entity.getVersion(),
- ChangeLogEntry.ChangeType.PROPERTY_DELETE, field);
- changeLog.add(cle);
- } else {
- cle.addVersion(entity.getVersion());
- }
- }
- }
-
- } else { // greater than or equal to minVersion
-
- for (Field field : entity.getFields()) {
-
- String key = field.getName() + field.getValue();
- ChangeLogEntry cle = writeMap.get(key);
- if (cle == null) {
- cle = new ChangeLogEntry(
- entity.getId(), entity.getVersion(),
- ChangeLogEntry.ChangeType.PROPERTY_WRITE, field);
- writeMap.put(key, cle);
- changeLog.add(cle);
- } else {
- cle.addVersion(entity.getVersion());
- }
- }
+ final Entity currentEntity = mvccEntity.getEntity().orNull();
+
+ //get all fields in the current field that aren't in the previous fields
+ final Set<String> deletedFields = Sets.difference( previousFieldNames, currentFieldNames );
+
+ changeLog.addDeletes( deletedFields );
+
+ for ( String addedField : currentFieldNames ) {
+ changeLog.addWrite( currentEntity.getField( addedField ) );
}
+
+
+ previousFieldNames = currentFieldNames;
+ }
+
+ //subtract off the the last set of fields from the entity
+ if(currentFieldNames != null) {
+ changeLog.clear( currentFieldNames );
}
+
+
return changeLog;
}
- }
+
+
+ /**
+ * Get all the fieldNames on this entity
+ */
+ private Set<String> getFieldNames( final Optional<Entity> entity ) {
+ if ( !entity.isPresent() ) {
+ return Collections.emptySet();
+ }
+
+
+ Collection<Field> fields = entity.get().getFields();
+
+ Set<String> fieldNames = new HashSet<>( fields.size() );
+
+ for ( final Field field : entity.get().getFields() ) {
+ fieldNames.add( field.getName() );
+ }
+
+ return fieldNames;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogImpl.java
new file mode 100644
index 0000000..e931a1f
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogImpl.java
@@ -0,0 +1,135 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.collection.mvcc.changelog;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.usergrid.persistence.model.field.Field;
+
+
+/**
+ * Create a new instance of the change log
+ */
+public class ChangeLogImpl implements ChangeLog {
+
+ private final HashMap<String, Field> additions;
+ private final HashSet<String> deletions;
+
+
+ /**
+ * @param approximateMaxFieldCount The estimated upper bounds of the changelog, helps with efficiency by
+ * overallocation if necessary
+ */
+ public ChangeLogImpl( final int approximateMaxFieldCount ) {
+ //todo, may be way too large
+ additions = new HashMap( approximateMaxFieldCount );
+ deletions = new HashSet( approximateMaxFieldCount );
+ }
+
+
+ /**
+ * Add change logs to this log
+ *
+ * @param deletes The set containing all string fields to delete
+ */
+ public void addDeletes( final Set<String> deletes ) {
+ this.deletions.addAll( deletes );
+ clearAdditions( deletes );
+ }
+
+
+ /**
+ * Add the field as a write column
+ */
+ public void addWrite( final Field field ) {
+ final String fieldName = field.getName();
+ this.additions.put( field.getName(), field );
+ this.deletions.remove( fieldName );
+ }
+
+
+ /**
+ * Remove the names from the additions and deletions
+ */
+ public void clear( final Set<String> names ) {
+ this.deletions.removeAll( names );
+ clearAdditions( names );
+ }
+
+
+ /**
+ * Clear additions by name
+ */
+ private void clearAdditions( final Set<String> deletes ) {
+
+ for ( final String key : deletes ) {
+ this.additions.remove( key );
+ }
+ }
+
+
+ /**
+ * Clear all all additions and deletions
+ */
+ public void clear() {
+ this.additions.clear();
+ this.deletions.clear();
+ }
+
+
+ @Override
+ public Set<String> getDeletes() {
+ return deletions;
+ }
+
+
+ @Override
+ public Collection<Field> getWrites() {
+ return additions.values();
+ }
+
+
+ @Override
+ public int getSize() {
+ return deletions.size() + additions.size();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java
index 9b7f437..f9c66cc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/load/Load.java
@@ -29,8 +29,10 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
+import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.collection.serialization.impl.EntityRepairImpl;
import org.apache.usergrid.persistence.collection.service.UUIDService;
-import org.apache.usergrid.persistence.collection.util.RepairUtil;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -53,16 +55,20 @@ public class Load implements Func1<CollectionIoEvent<Id>, Entity> {
private final UUIDService uuidService;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
+ private final EntityRepair entityRepair;
@Inject
- public Load( final UUIDService uuidService, final MvccEntitySerializationStrategy entitySerializationStrategy ) {
+ public Load( final UUIDService uuidService, final MvccEntitySerializationStrategy entitySerializationStrategy, final
+ SerializationFig serializationFig ) {
Preconditions.checkNotNull( entitySerializationStrategy, "entitySerializationStrategy is required" );
Preconditions.checkNotNull( uuidService, "uuidService is required" );
this.uuidService = uuidService;
this.entitySerializationStrategy = entitySerializationStrategy;
+ entityRepair = new EntityRepairImpl( entitySerializationStrategy, serializationFig );
+
}
//TODO: do reads partial merges in batches. maybe 5 or 10 at a time.
@@ -88,11 +94,18 @@ public class Load implements Func1<CollectionIoEvent<Id>, Entity> {
Iterator<MvccEntity> results = entitySerializationStrategy.load(
collectionScope, entityId, versionMax, 1 );
- MvccEntity repairedEntity = RepairUtil.repair( results,collectionScope,entitySerializationStrategy );
+ if(!results.hasNext()){
+ return null;
+ }
+
+ final MvccEntity returned = results.next();
+
+ final MvccEntity repairedEntity = entityRepair.maybeRepair( collectionScope, returned );
+
if(repairedEntity == null)
return null;
- return repairedEntity.getEntity().get();
+ return repairedEntity.getEntity().orNull();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java
new file mode 100644
index 0000000..c4fddd2
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java
@@ -0,0 +1,38 @@
+package org.apache.usergrid.persistence.collection.serialization;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+
+
+/**
+ * Interface for entity repair operations
+ */
+public interface EntityRepair {
+
+ /**
+ * Run the repair task for this entity. If the entity does not need repaired, it will just be returned
+ *
+ * @param collectionScope The scope of the entity to possibly repair
+ * @param targetEntity The entity to check and repair
+ * @return The source entity or the repaired entity
+ */
+ public MvccEntity maybeRepair(CollectionScope collectionScope, MvccEntity targetEntity);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
new file mode 100644
index 0000000..d26cd3f
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
@@ -0,0 +1,146 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
+import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLog;
+import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
+import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
+import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.Field;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+@Singleton
+public class EntityRepairImpl implements EntityRepair {
+
+
+ private static final ChangeLogGenerator CHANGE_LOG_GENERATOR = new ChangeLogGeneratorImpl();
+
+ private final MvccEntitySerializationStrategy mvccEntitySerializationStrategy;
+ private final SerializationFig serializationFig;
+
+
+ @Inject
+ public EntityRepairImpl( final MvccEntitySerializationStrategy mvccEntitySerializationStrategy,
+ final SerializationFig serializationFig ) {
+ this.mvccEntitySerializationStrategy = mvccEntitySerializationStrategy;
+ this.serializationFig = serializationFig;
+ }
+
+
+ @Override
+ public MvccEntity maybeRepair( final CollectionScope collectionScope, final MvccEntity targetEntity ) {
+ if ( !needsRepaired( targetEntity ) ) {
+ return targetEntity;
+ }
+
+
+ final List<MvccEntity> partialEntities = new ArrayList<>( serializationFig.getBufferSize() );
+
+ partialEntities.add( targetEntity );
+
+ final Iterator<MvccEntity> results = mvccEntitySerializationStrategy
+ .load( collectionScope, targetEntity.getId(), targetEntity.getVersion(),
+ serializationFig.getBufferSize() );
+
+
+
+ //discard the one that's equal to the version we were passed in
+ if(results.hasNext() ){
+ results.next();
+ }
+
+
+// MvccEntity oldestCompleteEntity;
+
+
+ while ( results.hasNext() ) {
+ final MvccEntity mvccEntity = results.next();
+ partialEntities.add( mvccEntity );
+
+
+ if ( !needsRepaired( mvccEntity ) ) {
+ break;
+ }
+ }
+
+ Collections.reverse( partialEntities );
+
+ final ChangeLog changeLog =
+ CHANGE_LOG_GENERATOR.getChangeLog( partialEntities );
+
+
+ //repair
+ final MvccEntity mergedEntity = entityRepair( changeLog, targetEntity );
+
+ try {
+ mvccEntitySerializationStrategy.write( collectionScope, mergedEntity ).execute();
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Couldn't rewrite repaired entity", e );
+ }
+
+ return mergedEntity;
+
+ }
+
+
+ /**
+ * Appies changes to the entity log, oldest to newest version
+ */
+ private MvccEntity entityRepair( final ChangeLog changeLog, final MvccEntity targetEntity ) {
+
+ //TODO review this, why do we care other than just replaying the changelog?
+
+ final Entity entity = targetEntity.getEntity().get();
+
+ for(final String removedField: changeLog.getDeletes()){
+ entity.removeField( removedField );
+ }
+
+ for(final Field newField : changeLog.getWrites()){
+ entity.setField( newField );
+ }
+
+ return targetEntity;
+ }
+
+
+ /**
+ * Returns true if the entity needs repaired
+ */
+ private boolean needsRepaired( final MvccEntity headEntity ) {
+
+ final MvccEntity.Status status = headEntity.getStatus();
+
+ return !(status == MvccEntity.Status.COMPLETE || status == MvccEntity.Status.DELETED || !headEntity.getEntity().isPresent());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
new file mode 100644
index 0000000..89ec69e
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
@@ -0,0 +1,69 @@
+package org.apache.usergrid.persistence.collection.serialization.impl;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public class EntitySetImpl implements EntitySet {
+
+
+ private final Map<Id, MvccEntity> entities;
+
+
+ public EntitySetImpl(
+ final int expectedSize ) {
+ this.entities = new HashMap<>( expectedSize );
+ }
+
+
+ public void addEntity( final MvccEntity entity ) {
+ entities.put( entity.getId(), entity );
+ }
+
+
+ @Override
+ public Entity getEntity( final Id entityId ) {
+ final MvccEntity entity = entities.get( entityId );
+
+ if ( entity == null ) {
+ return null;
+ }
+
+ return entity.getEntity().orNull();
+ }
+
+
+ @Override
+ public int size() {
+ return entities.size();
+ }
+
+
+ @Override
+ public boolean isEmpty() {
+ return entities.size() == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index 419d0f0..b36ee0e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -19,20 +19,29 @@ package org.apache.usergrid.persistence.collection.serialization.impl;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import java.util.UUID;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.ReversedType;
import org.apache.cassandra.db.marshal.UUIDType;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
+import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator;
import org.apache.usergrid.persistence.core.astyanax.ColumnParser;
@@ -45,7 +54,6 @@ import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.dataformat.smile.SmileFactory;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
@@ -57,16 +65,17 @@ import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ColumnList;
import com.netflix.astyanax.model.CompositeBuilder;
import com.netflix.astyanax.model.CompositeParser;
import com.netflix.astyanax.model.Composites;
+import com.netflix.astyanax.model.Row;
import com.netflix.astyanax.query.RowQuery;
import com.netflix.astyanax.serializers.AbstractSerializer;
import com.netflix.astyanax.serializers.ByteBufferSerializer;
import com.netflix.astyanax.serializers.BytesArraySerializer;
import com.netflix.astyanax.serializers.UUIDSerializer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
/**
* @author tnine
@@ -74,9 +83,9 @@ import org.slf4j.LoggerFactory;
@Singleton
public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializationStrategy, Migration {
- private static final Logger log = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
+ private static final Logger log = LoggerFactory.getLogger( MvccLogEntrySerializationStrategyImpl.class );
- private static final EntitySerializer SER = new EntitySerializer();
+ private static final EntitySerializer ENTITY_JSON_SER = new EntitySerializer();
private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
@@ -94,11 +103,15 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
protected final Keyspace keyspace;
+ protected final SerializationFig serializationFig;
+ protected final EntityRepair repair;
@Inject
- public MvccEntitySerializationStrategyImpl( final Keyspace keyspace ) {
+ public MvccEntitySerializationStrategyImpl( final Keyspace keyspace, final SerializationFig serializationFig ) {
this.keyspace = keyspace;
+ this.serializationFig = serializationFig;
+ this.repair = new EntityRepairImpl( this, serializationFig );
}
@@ -114,9 +127,10 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
try {
- colMutation.putColumn( colName,
- SER.toByteBuffer( new EntityWrapper( entity.getStatus(), entity.getEntity() ) ) );
- } catch ( Exception e ) {
+ colMutation.putColumn( colName, ENTITY_JSON_SER
+ .toByteBuffer( new EntityWrapper( entity.getStatus(), entity.getEntity() ) ) );
+ }
+ catch ( Exception e ) {
// throw better exception if we can
if ( entity != null || entity.getEntity().get() != null ) {
throw new CollectionRuntimeException( entity, collectionScope, e );
@@ -147,34 +161,79 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
return null;
}
catch ( ConnectionException e ) {
- throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra", e );
+ throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra",
+ e );
}
- return new MvccColumnParser(entityId).parseColumn(column);
+ return new MvccColumnParser( entityId ).parseColumn( column );
}
@Override
- public Iterator<MvccEntity> load( final CollectionScope collectionScope, final Id entityId, final UUID version,
- final int fetchSize ) {
+ public EntitySet load( final CollectionScope collectionScope, final Collection<Id> entityIds,
+ final UUID maxVersion ) {
+
Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
- Preconditions.checkNotNull( entityId, "entity id is required" );
- Preconditions.checkNotNull( version, "version is required" );
- Preconditions.checkArgument( fetchSize > 0, "max Size must be greater than 0" );
+ Preconditions.checkNotNull( entityIds, "entityIds is required" );
+ Preconditions.checkArgument( entityIds.size() > 0, "entityIds is required" );
+ Preconditions.checkNotNull( maxVersion, "version is required" );
+ final List<ScopedRowKey<CollectionScope, Id>> rowKeys = new ArrayList<>( entityIds.size() );
+
+
+ for ( final Id entityId : entityIds ) {
+ rowKeys.add( ScopedRowKey.fromKey( collectionScope, entityId ) );
+ }
+
+
+ final Iterator<Row<ScopedRowKey<CollectionScope, Id>, UUID>> latestEntityColumns;
+
+
+
+ try {
+ latestEntityColumns = keyspace.prepareQuery( CF_ENTITY_DATA ).getKeySlice( rowKeys )
+ .withColumnRange( maxVersion, null, false, 1 ).execute().getResult()
+ .iterator();
+ } catch ( ConnectionException e ) {
+ throw new CollectionRuntimeException( null, collectionScope, "An error occurred connecting to cassandra",
+ e );
+ }
+
+
+
+ final EntitySetImpl entitySetResults = new EntitySetImpl( entityIds.size() );
+
+ while ( latestEntityColumns.hasNext() ) {
+ final Row<ScopedRowKey<CollectionScope, Id>, UUID> row = latestEntityColumns.next();
+
+ final ColumnList<UUID> columns = row.getColumns();
+
+ if ( columns.size() == 0 ) {
+ continue;
+ }
- RowQuery<ScopedRowKey<CollectionScope, Id>, UUID> query = keyspace.prepareQuery(CF_ENTITY_DATA).getKey(ScopedRowKey.fromKey(collectionScope, entityId))
- .withColumnRange(version, null, false, fetchSize);
+ final Id entityId = row.getKey().getKey();
- return new ColumnNameIterator(query, new MvccColumnParser(entityId), false);
+ final Column<UUID> column = columns.getColumnByIndex( 0 );
+ final MvccEntity parsedEntity = new MvccColumnParser( entityId ).parseColumn( column );
+
+ //we *might* need to repair, it's not clear so check before loading into result sets
+ final MvccEntity maybeRepaired = repair.maybeRepair( collectionScope, parsedEntity );
+
+ entitySetResults.addEntity( maybeRepaired );
+
+ }
+
+ return entitySetResults;
}
+
@Override
- public Iterator<MvccEntity> loadHistory( final CollectionScope collectionScope, final Id entityId, final UUID version,
+ public Iterator<MvccEntity> load( final CollectionScope collectionScope, final Id entityId, final UUID version,
final int fetchSize ) {
Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
@@ -182,11 +241,29 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
Preconditions.checkNotNull( version, "version is required" );
Preconditions.checkArgument( fetchSize > 0, "max Size must be greater than 0" );
- RowQuery<ScopedRowKey<CollectionScope, Id>, UUID> query = keyspace.prepareQuery(CF_ENTITY_DATA).getKey(ScopedRowKey.fromKey(collectionScope, entityId))
- .withColumnRange(null, version, true, fetchSize);
- return new ColumnNameIterator(query, new MvccColumnParser(entityId), false);
+ RowQuery<ScopedRowKey<CollectionScope, Id>, UUID> query =
+ keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( ScopedRowKey.fromKey( collectionScope, entityId ) )
+ .withColumnRange( version, null, false, fetchSize );
+
+ return new ColumnNameIterator( query, new MvccColumnParser( entityId ), false );
+ }
+
+
+ @Override
+ public Iterator<MvccEntity> loadHistory( final CollectionScope collectionScope, final Id entityId,
+ final UUID version, final int fetchSize ) {
+
+ Preconditions.checkNotNull( collectionScope, "collectionScope is required" );
+ Preconditions.checkNotNull( entityId, "entity id is required" );
+ Preconditions.checkNotNull( version, "version is required" );
+ Preconditions.checkArgument( fetchSize > 0, "max Size must be greater than 0" );
+
+ RowQuery<ScopedRowKey<CollectionScope, Id>, UUID> query =
+ keyspace.prepareQuery( CF_ENTITY_DATA ).getKey( ScopedRowKey.fromKey( collectionScope, entityId ) )
+ .withColumnRange( null, version, true, fetchSize );
+ return new ColumnNameIterator( query, new MvccColumnParser( entityId ), false );
}
@@ -201,8 +278,8 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
return doWrite( collectionScope, entityId, new RowOp() {
@Override
public void doOp( final ColumnListMutation<UUID> colMutation ) {
- colMutation.putColumn( version, SER.toByteBuffer(
- new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() ) ) );
+ colMutation.putColumn( version, ENTITY_JSON_SER
+ .toByteBuffer( new EntityWrapper( MvccEntity.Status.COMPLETE, Optional.<Entity>absent() ) ) );
}
} );
}
@@ -229,8 +306,10 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
//create the CF entity data. We want it reversed b/c we want the most recent version at the top of the
//row for fast seeks
- MultiTennantColumnFamilyDefinition cf = new MultiTennantColumnFamilyDefinition( CF_ENTITY_DATA,
- BytesType.class.getSimpleName(), ReversedType.class.getSimpleName() + "(" + UUIDType.class.getSimpleName() + ")", BytesType.class.getSimpleName(), MultiTennantColumnFamilyDefinition.CacheOption.KEYS );
+ MultiTennantColumnFamilyDefinition cf =
+ new MultiTennantColumnFamilyDefinition( CF_ENTITY_DATA, BytesType.class.getSimpleName(),
+ ReversedType.class.getSimpleName() + "(" + UUIDType.class.getSimpleName() + ")",
+ BytesType.class.getSimpleName(), MultiTennantColumnFamilyDefinition.CacheOption.KEYS );
return Collections.singleton( cf );
@@ -249,7 +328,6 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
}
-
/**
* Simple callback to perform puts and deletes with a common row setup code
*/
@@ -276,23 +354,24 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
}
}
+
/**
* Converts raw columns the to MvccEntity representation
-
*/
private static final class MvccColumnParser implements ColumnParser<UUID, MvccEntity> {
private final Id id;
- private MvccColumnParser(Id id) {
+
+ private MvccColumnParser( Id id ) {
this.id = id;
}
@Override
- public MvccEntity parseColumn(Column<UUID> column) {
+ public MvccEntity parseColumn( Column<UUID> column ) {
- final EntityWrapper deSerialized = column.getValue( SER );
+ final EntityWrapper deSerialized = column.getValue( ENTITY_JSON_SER );
//Inject the id into it.
if ( deSerialized.entity.isPresent() ) {
@@ -303,11 +382,12 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
}
}
+
public static class EntitySerializer extends AbstractSerializer<EntityWrapper> {
public static final EntitySerializer INSTANCE = new EntitySerializer();
- public static final SmileFactory f = new SmileFactory( );
+ public static final SmileFactory f = new SmileFactory();
public static ObjectMapper mapper;
@@ -321,16 +401,20 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
//the marker for when we're passed a "null" value
private static final byte[] EMPTY = new byte[] { 0x0 };
+
public EntitySerializer() {
try {
mapper = new ObjectMapper( f );
-// mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output, causes slowness
- mapper.enableDefaultTypingAsProperty(ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class");
- } catch ( Exception e ) {
- throw new RuntimeException("Error setting up mapper", e);
+ // mapper.enable(SerializationFeature.INDENT_OUTPUT); don't indent output,
+ // causes slowness
+ mapper.enableDefaultTypingAsProperty( ObjectMapper.DefaultTyping.JAVA_LANG_OBJECT, "@class" );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Error setting up mapper", e );
}
}
+
@Override
public ByteBuffer toByteBuffer( final EntityWrapper wrapper ) {
if ( wrapper == null ) {
@@ -363,15 +447,16 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
builder.addBytes( mapper.writeValueAsBytes( wrapper.entity.get() ) );
}
catch ( Exception e ) {
- throw new RuntimeException("Unable to serialize entity", e);
+ throw new RuntimeException( "Unable to serialize entity", e );
}
return builder.build();
}
+
@Override
public EntityWrapper fromByteBuffer( final ByteBuffer byteBuffer ) {
- CompositeParser parser = Composites.newCompositeParser( byteBuffer );
+ CompositeParser parser = Composites.newCompositeParser( byteBuffer );
byte[] version = parser.read( BYTES_ARRAY_SERIALIZER );
@@ -389,7 +474,7 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
Entity storedEntity = null;
- ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER );
+ ByteBuffer jsonBytes = parser.read( BUFFER_SERIALIZER );
byte[] array = jsonBytes.array();
int start = jsonBytes.arrayOffset();
int length = jsonBytes.remaining();
@@ -398,10 +483,10 @@ public class MvccEntitySerializationStrategyImpl implements MvccEntitySerializat
storedEntity = mapper.readValue( array, start, length, Entity.class );
}
catch ( Exception e ) {
- throw new RuntimeException("Unable to read entity data", e);
+ throw new RuntimeException( "Unable to read entity data", e );
}
- final Optional<Entity> entity = Optional.of( storedEntity);
+ final Optional<Entity> entity = Optional.of( storedEntity );
if ( Arrays.equals( STATE_COMPLETE, state ) ) {
return new EntityWrapper( MvccEntity.Status.COMPLETE, entity );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/RepairUtil.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/RepairUtil.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/RepairUtil.java
deleted file mode 100644
index aac5794..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/util/RepairUtil.java
+++ /dev/null
@@ -1,139 +0,0 @@
-package org.apache.usergrid.persistence.collection.util;
-
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogEntry;
-import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
-import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-
-
-/**
- * This class holds different methods pertaining to the consolidation of an mvccentity.
- *
- */
-public class RepairUtil {
-
- private static final ChangeLogGenerator changeLogGenerator = new ChangeLogGeneratorImpl();
-
-
- /**
- * Name of the operation to be done on the entity. Repair starts the process for determining whether an
- * entity needs repair or not.
- * @param results
- * @param collectionScope
- * @param entitySerializationStrategy
- * @return
- */
- public static MvccEntity repair( Iterator<MvccEntity> results, CollectionScope collectionScope,
- MvccEntitySerializationStrategy entitySerializationStrategy ) {
-
- //nothing to do, we didn't get a result back
- if ( !results.hasNext() ) {
- return null;
- }
-
- final MvccEntity partialEntity = results.next();
- List<MvccEntity> partialEntities = new ArrayList<>();
-
- //this entity has been marked as cleared.(deleted)
- //The version exists, but does not have entity data
- if ( !partialEntity.getEntity().isPresent() && ( partialEntity.getStatus() == MvccEntity.Status.DELETED
- || partialEntity.getStatus() == MvccEntity.Status.COMPLETE ) ) {
- return null;
- }
-
- if ( partialEntity.getStatus() == MvccEntity.Status.COMPLETE ) {
- return partialEntity;
- }
-
- partialEntities.add( partialEntity );
-
-
- while ( results.hasNext() ) {
- final MvccEntity previousEntity = results.next();
- partialEntities.add( previousEntity );
-
- if ( previousEntity.getStatus() != MvccEntity.Status.PARTIAL ) {
- return repairAndWrite( partialEntities, partialEntity, entitySerializationStrategy, collectionScope );
- }
- }
- return null;
- }
-
-
- /**
- * Repairs the entity then overwrites the previous entity to become the new completed entity.
- * @param partialEntities
- * @param targetEntity The entity that should ultimately contain all merged data
- * @param entitySerializationStrategy
- * @param collectionScope
- * @return
- */
- private static MvccEntity repairAndWrite( List<MvccEntity> partialEntities, MvccEntity targetEntity,
- MvccEntitySerializationStrategy entitySerializationStrategy,
- CollectionScope collectionScope ) {
- Collections.reverse( partialEntities );
-
- //repair
- final MvccEntity mergedEntity = entityRepair( changeLogGenerator.getChangeLog( partialEntities.iterator(),
- partialEntities.get( partialEntities.size() - 1 ).getVersion() ), partialEntities, targetEntity
- );
-
- try {
- entitySerializationStrategy.write( collectionScope, mergedEntity ).execute();
- }
- catch ( Exception e ) {
- throw new RuntimeException( "Couldn't rewrite repaired entity", e );
- }
- return mergedEntity;
- }
-
-
- /**
- * Applies the changelog to the completed entity.
- * @param changeLogEntryList
- * @param results
- * @param completedEntity
- * @return
- */
- private static MvccEntity entityRepair( List<ChangeLogEntry> changeLogEntryList, List<MvccEntity> results,
- MvccEntity completedEntity ) {
- int changeLogIndex = 0;
- for ( MvccEntity result : results ) {
-
- while ( changeLogIndex != changeLogEntryList.size() ) {
-
- ChangeLogEntry changeLogEntry = changeLogEntryList.get( changeLogIndex );
-
- if ( result.getId().equals( changeLogEntry.getEntryId() ) ) {
-
- ChangeLogEntry.ChangeType changeType = changeLogEntry.getChangeType();
-
- if ( changeType.equals( ChangeLogEntry.ChangeType.PROPERTY_DELETE ) ) {
- completedEntity.getEntity().get().getFields().remove( changeLogEntry.getField() );
- }
- else if ( changeType.equals( ChangeLogEntry.ChangeType.PROPERTY_WRITE ) ) {
- completedEntity.getEntity().get().setField( changeLogEntry.getField() );
- }
- changeLogIndex++;
- }
- else {
- break;
- }
- }
- }
-
- if ( !completedEntity.getEntity().isPresent() ) {
- return null;
- }
-
- return completedEntity;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f7d406e3/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
index 56792a9..6795893 100644
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImplTest.java
@@ -18,215 +18,261 @@
package org.apache.usergrid.persistence.collection.mvcc.changelog;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Iterator;
-import java.util.List;
+import java.util.Set;
-import org.jukito.JukitoModule;
-import org.jukito.UseModules;
-import org.junit.Assert;
-import org.junit.ClassRule;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.EntityCollectionManager;
-import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
-import org.apache.usergrid.persistence.collection.guice.CollectionModule;
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.collection.guice.TestCollectionModule;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
-import org.apache.usergrid.persistence.core.cassandra.ITRunner;
+import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
+import org.apache.usergrid.persistence.model.field.BooleanField;
+import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.field.IntegerField;
import org.apache.usergrid.persistence.model.field.StringField;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import com.google.inject.Inject;
+import com.google.common.base.Optional;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import rx.Observable;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
/**
* Test basic operation of change log
*/
-@RunWith( ITRunner.class )
-@UseModules( TestCollectionModule.class )
public class ChangeLogGeneratorImplTest {
private static final Logger LOG = LoggerFactory.getLogger( ChangeLogGeneratorImplTest.class );
- @Inject
- @Rule
- public MigrationManagerRule migrationManagerRule;
+ /**
+ * Test rolling up 3 versions, properties are added then deleted
+ */
+ @Test
+ public void testBasicOperation() throws ConnectionException {
+
+ LOG.info( "ChangeLogGeneratorImpl test" );
+
+
+ final Id entityId = new SimpleId( "test" );
+
+ Entity e1 = new Entity( entityId );
+ e1.setField( new StringField( "name", "name1" ) );
+ e1.setField( new IntegerField( "count", 1 ) );
+ e1.setField( new BooleanField( "single", true ) );
+
+ final MvccEntity mvccEntity1 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.COMPLETE, e1 );
+
+ Entity e2 = new Entity( entityId );
+ e2.setField( new StringField( "name", "name2" ) );
+ e2.setField( new IntegerField( "count", 2 ) );
+ e2.setField( new StringField( "nickname", "buddy" ) );
+ e2.setField( new BooleanField( "cool", false ) );
+
+ final MvccEntity mvccEntity2 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, e2 );
+
+
+ Entity e3 = new Entity( entityId );
+ e3.setField( new StringField( "name", "name3" ) );
+ e3.setField( new IntegerField( "count", 2 ) );
+ //appears in e1, since it's been added again, we want to make sure it doesn't appear in the delete list
+ e3.setField( new BooleanField( "single", true ) );
+
+ final MvccEntity mvccEntity3 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, e3 );
+
+
+ ChangeLogGeneratorImpl instance = new ChangeLogGeneratorImpl();
+ ChangeLog result =
+ instance.getChangeLog( Arrays.asList( mvccEntity1, mvccEntity2, mvccEntity3 ) ); // minVersion = e3
+
+
+ assertEquals( "All changes not present", 2, result.getSize() );
+
- @Inject
- private EntityCollectionManagerFactory factory;
+ Collection<Field> changes = result.getWrites();
+
+ assertEquals( 0, changes.size() );
+
+ Set<String> deletes = result.getDeletes();
+
+ assertEquals( 2, deletes.size() );
+
+ assertTrue( deletes.contains( "nickname" ) );
+ assertTrue( deletes.contains( "cool" ) );
+ }
- @Inject
- MvccEntitySerializationStrategy mvccEntitySerializationStrategy;
/**
- * Test that change log creation follows Todd's example.
- * TODO, can we do this without doing serialization I/O on the entities?
- * This seems out of the scope of the changelog itself
+ * Test rolling up 3 versions, properties are added then deleted
*/
@Test
- public void testBasicOperation() throws ConnectionException {
+ public void testDeletedVersionFirst() throws ConnectionException {
+
+ LOG.info( "ChangeLogGeneratorImpl test" );
+
+
+ final Id entityId = new SimpleId( "test" );
+
+ final MvccEntity mvccEntity1 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.DELETED,
+ Optional.<Entity>absent() );
+
+ Entity e2 = new Entity( entityId );
+ e2.setField( new StringField( "name", "name2" ) );
+ e2.setField( new IntegerField( "count", 2 ) );
+ e2.setField( new StringField( "nickname", "buddy" ) );
+ e2.setField( new BooleanField( "cool", false ) );
+
+ final MvccEntity mvccEntity2 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, e2 );
+
+
+ Entity e3 = new Entity( entityId );
+ e3.setField( new StringField( "name", "name3" ) );
+ e3.setField( new IntegerField( "count", 2 ) );
+ //appears in e1, since it's been added again, we want to make sure it doesn't appear in the delete list
+ e3.setField( new BooleanField( "single", true ) );
+
+ final MvccEntity mvccEntity3 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, e3 );
+
+
+ ChangeLogGeneratorImpl instance = new ChangeLogGeneratorImpl();
+ ChangeLog result =
+ instance.getChangeLog( Arrays.asList( mvccEntity1, mvccEntity2, mvccEntity3 ) ); // minVersion = e3
+
+
+ assertEquals( "All changes not present", 2, result.getSize() );
+
- LOG.info("ChangeLogGeneratorImpl test");
+ Collection<Field> changes = result.getWrites();
- // create an entity and make a series of changes to it so that versions get created
- CollectionScope context = new CollectionScopeImpl(
- new SimpleId( "organization" ), new SimpleId( "test" ), "test" );
+ assertEquals( 0, changes.size() );
- // Todd's example:
- //
- // V1 : { "name" : "name1" , "count": 1}
- // V2: { "name" : "name2" , "count": 2, "nickname" : "buddy"}
- // V3: { "name" : "name3" , "count": 2}
-
- EntityCollectionManager manager = factory.createCollectionManager( context );
- Entity e1 = new Entity( new SimpleId( "test" ) );
+
+
+ Set<String> deletes = result.getDeletes();
+
+ assertEquals( 2, deletes.size() );
+
+ assertTrue( deletes.contains( "nickname" ) );
+ assertTrue( deletes.contains( "cool" ) );
+ }
+
+
+ /**
+ * Test rolling up 3 versions, properties are added then deleted
+ */
+ @Test
+ public void testDeletedMiddle() throws ConnectionException {
+
+ LOG.info( "ChangeLogGeneratorImpl test" );
+
+
+ final Id entityId = new SimpleId( "test" );
+
+ Entity e1 = new Entity( entityId );
e1.setField( new StringField( "name", "name1" ) );
e1.setField( new IntegerField( "count", 1 ) );
- Observable<Entity> o1 = manager.write( e1 );
- e1 = o1.toBlocking().lastOrDefault( null );
+ e1.setField( new BooleanField( "single", true ) );
+
+ final MvccEntity mvccEntity1 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.COMPLETE, e1 );
- Entity e2 = manager.load( e1.getId() ).toBlocking().lastOrDefault( null );
+ Entity e2 = new Entity( entityId );
e2.setField( new StringField( "name", "name2" ) );
e2.setField( new IntegerField( "count", 2 ) );
e2.setField( new StringField( "nickname", "buddy" ) );
- Observable<Entity> o2 = manager.write( e2 );
- e2 = o2.toBlocking().lastOrDefault( null );
+ e2.setField( new BooleanField( "cool", false ) );
+
+ final MvccEntity mvccEntity2 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.DELETED, e2 );
- Entity e3 = manager.load( e1.getId() ).toBlocking().lastOrDefault( null );
+
+ Entity e3 = new Entity( entityId );
e3.setField( new StringField( "name", "name3" ) );
e3.setField( new IntegerField( "count", 2 ) );
- e3.getFields().remove(new StringField( "nickname", "buddy"));
- Observable<Entity> o3 = manager.write( e3 );
- e3 = o3.toBlocking().lastOrDefault( null );
-
- {
- // test minVersion of e3
- //
- // based on that data we expect something like this:
- //
- // Type = PROPERTY_WRITE, Property = count, Value = 2, Versions = [560c7e10-a925-11e3-bf9d-10ddb1de66c4]
- // Type = PROPERTY_WRITE, Property = name, Value = name3, Versions = [560c7e10-a925-11e3-bf9d-10ddb1de66c4]
- //
- // Type = PROPERTY_DELETE, Property = nickname, Value = buddy, Versions = [560b6c9e-a925-11e3-bf9d-10ddb1de66c4]
- // Type = PROPERTY_DELETE, Property = name, Value = name2, Versions = [560b6c9e-a925-11e3-bf9d-10ddb1de66c4]
- // Type = PROPERTY_DELETE, Property = count, Value = 1, Versions = [55faa3bc-a925-11e3-bf9d-10ddb1de66c4]
- // Type = PROPERTY_DELETE, Property = name, Value = name1, Versions = [55faa3bc-a925-11e3-bf9d-10ddb1de66c4]
-
- Iterator<MvccEntity> versions = mvccEntitySerializationStrategy
- .load( context, e1.getId(), e3.getVersion(), 10);
-
- ChangeLogGeneratorImpl instance = new ChangeLogGeneratorImpl();
- List<ChangeLogEntry> result = instance.getChangeLog( versions, e3.getVersion() ); // minVersion = e3
-
- for (ChangeLogEntry cle : result) {
- LOG.info( cle.toString() );
- Assert.assertFalse( cle.getVersions().isEmpty() );
- }
-
- Assert.assertEquals( 6, result.size() );
- Assert.assertTrue( isAscendingOrder( result ) );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_WRITE, result.get( 0 ).getChangeType() );
- Assert.assertEquals( "count", result.get( 0 ).getField().getName() );
- Assert.assertEquals( "2", result.get( 0 ).getField().getValue().toString() );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_WRITE, result.get( 1 ).getChangeType() );
- Assert.assertEquals( "name", result.get( 1 ).getField().getName() );
- Assert.assertEquals( "name3", result.get( 1 ).getField().getValue() );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_DELETE, result.get( 2 ).getChangeType() );
- Assert.assertEquals( "nickname", result.get( 2 ).getField().getName() );
- Assert.assertEquals( "buddy", result.get( 2 ).getField().getValue() );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_DELETE, result.get( 3 ).getChangeType() );
- Assert.assertEquals( "name", result.get( 3 ).getField().getName() );
- Assert.assertEquals( "name2", result.get( 3 ).getField().getValue() );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_DELETE, result.get( 4 ).getChangeType() );
- Assert.assertEquals( "count", result.get( 4 ).getField().getName() );
- Assert.assertEquals( "1", result.get( 4 ).getField().getValue().toString() );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_DELETE, result.get( 5 ).getChangeType() );
- Assert.assertEquals( "name", result.get( 5 ).getField().getName() );
- Assert.assertEquals( "name1", result.get( 5 ).getField().getValue() );
- }
-
- {
-
- // test minVersion of e2
- //
- // based on that data we expect something like this:
- //
- // Type = PROPERTY_WRITE, Property = name, Value = name3, Versions = [c771f63f-a927-11e3-8bfc-10ddb1de66c4]
- // Type = PROPERTY_WRITE, Property = count, Value = 2, Versions = [c770e4cd-a927-11e3-8bfc-10ddb1de66c4, c771f63f-a927-11e3-8bfc-10ddb1de66c4]
- // Type = PROPERTY_WRITE, Property = nickname, Value = buddy, Versions = [c770e4cd-a927-11e3-8bfc-10ddb1de66c4]
- // Type = PROPERTY_WRITE, Property = name, Value = name2, Versions = [c770e4cd-a927-11e3-8bfc-10ddb1de66c4]
-
- // Type = PROPERTY_DELETE, Property = count, Value = 1, Versions = [c75f589b-a927-11e3-8bfc-10ddb1de66c4]
- // Type = PROPERTY_DELETE, Property = name, Value = name1, Versions = [c75f589b-a927-11e3-8bfc-10ddb1de66c4]
-
- Iterator<MvccEntity> versions = mvccEntitySerializationStrategy
- .load( context, e1.getId(), e3.getVersion(), 10);
-
- ChangeLogGeneratorImpl instance = new ChangeLogGeneratorImpl();
- List<ChangeLogEntry> result = instance.getChangeLog( versions, e2.getVersion() ); // minVersion = e2
-
- for (ChangeLogEntry cle : result) {
- LOG.info( cle.toString() );
- Assert.assertFalse( cle.getVersions().isEmpty() );
- }
- Assert.assertEquals(6, result.size() );
- Assert.assertTrue( isAscendingOrder( result ) );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_WRITE, result.get( 2 ).getChangeType() );
- Assert.assertEquals( "nickname", result.get( 2 ).getField().getName() );
- Assert.assertEquals( "buddy", result.get( 2 ).getField().getValue() );
-
- Assert.assertEquals( ChangeLogEntry.ChangeType.PROPERTY_DELETE, result.get( 4 ).getChangeType() );
- Assert.assertEquals( "count", result.get( 4 ).getField().getName() );
- Assert.assertEquals( "1", result.get( 4 ).getField().getValue().toString() );
- }
- }
+ //appears in e1, since it's been added again, we want to make sure it doesn't appear in the delete list
+ e3.setField( new BooleanField( "single", true ) );
+
+ final MvccEntity mvccEntity3 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, e3 );
+
+
+ ChangeLogGeneratorImpl instance = new ChangeLogGeneratorImpl();
+ ChangeLog result =
+ instance.getChangeLog( Arrays.asList( mvccEntity1, mvccEntity2, mvccEntity3 ) ); // minVersion = e3
+
+
+ assertEquals( "All changes present", 0, result.getSize() );
+
+
+ Collection<Field> changes = result.getWrites();
- public static boolean isAscendingOrder( Collection<ChangeLogEntry> col ) {
- Comparable previous = null;
- for ( Comparable item : col ) {
- if ( previous == null ) {
- previous = item;
- continue;
- }
- int comparedToPrevious = item.compareTo( previous );
- if ( comparedToPrevious < 0 ) {
- return false;
- }
- }
- return true;
+ assertEquals( 0, changes.size() );
+
+ Set<String> deletes = result.getDeletes();
+
+ assertEquals( 0, deletes.size() );
}
- @SuppressWarnings( "UnusedDeclaration" )
- public static class TestModule extends JukitoModule {
+ /**
+ * Test rolling up 3 versions, properties are added then deleted
+ */
+ @Test
+ public void testDeletedLast() throws ConnectionException {
+
+ final Id entityId = new SimpleId( "test" );
+
+ Entity e1 = new Entity( entityId );
+ e1.setField( new StringField( "name", "name1" ) );
+ e1.setField( new IntegerField( "count", 1 ) );
+ e1.setField( new BooleanField( "single", true ) );
+
+ final MvccEntity mvccEntity1 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.COMPLETE, e1 );
+
+ Entity e2 = new Entity( entityId );
+ e2.setField( new StringField( "name", "name2" ) );
+ e2.setField( new IntegerField( "count", 2 ) );
+ e2.setField( new StringField( "nickname", "buddy" ) );
+ e2.setField( new BooleanField( "cool", false ) );
+
+ final MvccEntity mvccEntity2 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.PARTIAL, e2 );
+
+
+ final MvccEntity mvccEntity3 =
+ new MvccEntityImpl( entityId, UUIDGenerator.newTimeUUID(), MvccEntity.Status.DELETED,
+ Optional.<Entity>absent() );
+
+
+ ChangeLogGeneratorImpl instance = new ChangeLogGeneratorImpl();
+ ChangeLog result =
+ instance.getChangeLog( Arrays.asList( mvccEntity1, mvccEntity2, mvccEntity3 ) ); // minVersion = e3
+
+
+ assertEquals( "All changes not present", 0, result.getSize() );
+
+
+ Collection<Field> changes = result.getWrites();
+
+ assertEquals( 0, changes.size() );
+
+ Set<String> deletes = result.getDeletes();
- @Override
- protected void configureTest() {
- install( new CollectionModule() );
- }
+ assertEquals( 0, deletes.size() );
}
}
[11/16] git commit: merged
Posted by to...@apache.org.
merged
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a61e146c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a61e146c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a61e146c
Branch: refs/heads/two-dot-o
Commit: a61e146c4de34398c3d74cca0ca4ec42bc73fe23
Parents: 0d508f9 b3515f4
Author: Shawn Feldman <sf...@apache.org>
Authored: Thu Oct 9 14:00:22 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Thu Oct 9 14:00:22 2014 -0600
----------------------------------------------------------------------
.../impl/EntityCollectionManagerImpl.java | 48 ++++++++++++++++----
1 file changed, 40 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a61e146c/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index fab4269,32f214f..a1b9a13
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -218,44 -209,39 +219,54 @@@ public class EntityCollectionManagerImp
@Override
- public Observable<EntitySet> load( final Collection<Id> entityIds ) {
+ public Observable<EntitySet> load(final Collection<Id> entityIds) {
+
+ Preconditions.checkNotNull(entityIds, "entityIds cannot be null");
- final EntitySet results =
- entitySerializationStrategy.load(collectionScope, entityIds, UUIDGenerator.newTimeUUID());
- Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
- return Observable.just(results);
- return Observable.create( new Observable.OnSubscribe<EntitySet>() {
++ return Observable.create(new Observable.OnSubscribe<EntitySet>() {
+
+ @Override
- public void call( final Subscriber<? super EntitySet> subscriber ) {
++ public void call(final Subscriber<? super EntitySet> subscriber) {
+ try {
+ final EntitySet results =
- entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
++ entitySerializationStrategy.load(collectionScope, entityIds, UUIDGenerator.newTimeUUID());
+
- subscriber.onNext( results );
++ subscriber.onNext(results);
+ subscriber.onCompleted();
- }
- catch ( Exception e ) {
- subscriber.onError( e );
++ } catch (Exception e) {
++ subscriber.onError(e);
+ }
+ }
- } );
-
-
-
++ });
}
+ @Override
+ public Observable<Id> getIdField(final Field field) {
+ final List<Field> fields = Collections.singletonList(field);
- rx.Observable<Id> o = rx.Observable.from(fields).map(new Func1<Field, Id>() {
++ return rx.Observable.from(fields).map(new Func1<Field, Id>() {
+ @Override
+ public Id call(Field field) {
+ try {
+ UniqueValueSet set = uniqueValueSerializationStrategy.load(collectionScope, fields);
+ UniqueValue value = set.getValue(field.getName());
- Id id = value == null ? null :value.getEntityId();
++ Id id = value == null ? null : value.getEntityId();
+ return id;
+ } catch (ConnectionException e) {
+ log.error("Failed to getIdField", e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
- return o;
+ }
-
@Override
- public Observable<Entity> update( final Entity entity ) {
+ public Observable<Entity> update(final Entity entity) {
- log.debug( "Starting update process" );
+ log.debug("Starting update process");
//do our input validation
- Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
+ Preconditions.checkNotNull(entity, "Entity is required in the new stage of the mvcc write");
final Id entityId = entity.getId();
@@@ -314,13 -300,26 +325,34 @@@
@Override
++<<<<<<< HEAD
+ public Observable<VersionSet> getLatestVersion(
+ Collection<Id> entityIds) {
+
+ VersionSet logEntries = mvccLogEntrySerializationStrategy.load(collectionScope, entityIds,
+ UUIDGenerator.newTimeUUID());
++=======
+ public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
+
+ return Observable.create( new Observable.OnSubscribe<VersionSet>() {
+
+ @Override
+ public void call( final Subscriber<? super VersionSet> subscriber ) {
+ try {
+ final VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
+ UUIDGenerator.newTimeUUID() );
+
+ subscriber.onNext( logEntries );
+ subscriber.onCompleted();
+ }
+ catch ( Exception e ) {
+ subscriber.onError( e );
+ }
+ }
+ } );
+
++>>>>>>> b3515f45cdfdcc0eedd6e667701f69eccad37a7d
- return Observable.just(logEntries);
}
}
[07/16] git commit: Refactor of loader started
Posted by to...@apache.org.
Refactor of loader started
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7af7e702
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7af7e702
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7af7e702
Branch: refs/heads/two-dot-o
Commit: 7af7e702ef5ed5cc80926d2969c866df2d5d40aa
Parents: 63094ab
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 13:28:39 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 13:28:39 2014 -0600
----------------------------------------------------------------------
.../collection/EntityCollectionManager.java | 8 +-
.../persistence/collection/EntitySet.java | 2 -
.../persistence/collection/MvccEntity.java | 68 ++++++++++
.../persistence/collection/MvccLogEntry.java | 94 ++++++++++++++
.../persistence/collection/VersionSet.java | 67 ++++++++++
.../exception/CollectionRuntimeException.java | 2 +-
.../exception/WriteCommitException.java | 4 +-
.../WriteOptimisticVerifyException.java | 4 +-
.../exception/WriteStartException.java | 4 +-
.../exception/WriteUniqueVerifyException.java | 2 +-
.../collection/guice/CollectionModule.java | 2 +-
.../impl/EntityCollectionManagerImpl.java | 103 ++++++++-------
.../impl/EntityVersionCleanupTask.java | 2 +-
.../mvcc/MvccEntitySerializationStrategy.java | 3 +-
.../mvcc/MvccLogEntrySerializationStrategy.java | 17 ++-
.../mvcc/changelog/ChangeLogGenerator.java | 5 +-
.../mvcc/changelog/ChangeLogGeneratorImpl.java | 2 +-
.../collection/mvcc/entity/MvccEntity.java | 68 ----------
.../collection/mvcc/entity/MvccLogEntry.java | 93 -------------
.../mvcc/entity/MvccValidationUtils.java | 1 +
.../mvcc/entity/impl/MvccEntityDeleteEvent.java | 2 +-
.../mvcc/entity/impl/MvccEntityEvent.java | 2 +-
.../mvcc/entity/impl/MvccEntityImpl.java | 2 +-
.../mvcc/entity/impl/MvccEntityWriteEvent.java | 3 +-
.../mvcc/entity/impl/MvccLogEntryImpl.java | 2 +-
.../mvcc/stage/EntityUpdateEvent.java | 1 -
.../mvcc/stage/delete/MarkCommit.java | 4 +-
.../collection/mvcc/stage/delete/MarkStart.java | 4 +-
.../mvcc/stage/write/RollbackAction.java | 2 +-
.../mvcc/stage/write/WriteCommit.java | 4 +-
.../mvcc/stage/write/WriteOptimisticVerify.java | 44 +++----
.../collection/mvcc/stage/write/WriteStart.java | 5 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 2 +-
.../collection/serialization/EntityRepair.java | 2 +-
.../serialization/OptimisticUpdate.java | 2 +-
.../serialization/impl/EntityRepairImpl.java | 2 +-
.../serialization/impl/EntitySetImpl.java | 3 +-
.../serialization/impl/LogEntryIterator.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 3 +-
.../MvccLogEntrySerializationStrategyImpl.java | 129 +++++++++++++------
.../serialization/impl/VersionSetImpl.java | 80 ++++++++++++
.../collection/EntityCollectionManagerIT.java | 124 ++++++++++--------
.../changelog/ChangeLogGeneratorImplTest.java | 3 +-
.../mvcc/entity/impl/MvccEntityImplTest.java | 2 +-
.../mvcc/entity/impl/MvccLogEntryImplTest.java | 2 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 2 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 6 +-
.../mvcc/stage/delete/MarkStartTest.java | 4 +-
.../mvcc/stage/write/WriteCommitTest.java | 4 +-
.../stage/write/WriteOptimisticVerifyTest.java | 4 +-
.../mvcc/stage/write/WriteStartTest.java | 4 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 2 +-
.../serialization/EntityRepairImplTest.java | 2 +-
.../impl/LogEntryIteratorTest.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 2 +-
.../impl/MvccLESSTransientTest.java | 7 +-
...ccLogEntrySerializationStrategyImplTest.java | 19 +--
.../util/InvalidMvccEntityGenerator.java | 2 +-
.../util/InvalidValueGeneratorTest.java | 2 +-
.../collection/util/LogEntryMock.java | 4 +-
61 files changed, 622 insertions(+), 428 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
index 2625078..41d19e6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntityCollectionManager.java
@@ -55,10 +55,9 @@ public interface EntityCollectionManager {
public Observable<Entity> load( Id entityId );
/**
- * Return the latest version of the specified entity.
+ * Return the latest versions of the specified entityIds
*/
- public Observable<UUID> getLatestVersion( Id entityId );
- //TODO TN Change load to use multiget and return multiple entities. Only supports loading 1k per load operation.
+ public Observable<VersionSet> getLatestVersion( Collection<Id> entityId );
/**
* Load all the entityIds into the observable entity set
@@ -68,9 +67,6 @@ public interface EntityCollectionManager {
public Observable<EntitySet> load(Collection<Id> entityIds);
- //TODO Dave add a load versions using a multiget that will return a latest version structure for a collection of entity Ids
-
-
/**
* Takes the change and reloads an entity with all changes applied in this entity applied.
* The resulting entity from calling load will be the previous version of this entity + the entity
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
index 7803a04..1e811ae 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/EntitySet.java
@@ -18,8 +18,6 @@ package org.apache.usergrid.persistence.collection;/*
*/
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java
new file mode 100644
index 0000000..813d1f3
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccEntity.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection;
+
+
+import org.apache.usergrid.persistence.core.entity.EntityVersion;
+import org.apache.usergrid.persistence.model.entity.Entity;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * An entity with internal information for versioning
+ */
+public interface MvccEntity extends EntityVersion{
+
+ /**
+ * The possible State of the mvccEntity
+ */
+ public enum Status {
+
+ /**
+ * The entity being written represents a complete entity
+ */
+ COMPLETE,
+
+ /**
+ * The entity being written represents a partial entity
+ */
+ PARTIAL,
+
+ /**
+ * This entity has been marked as deleted
+ */
+ DELETED
+ ;
+ }
+
+
+ /**
+ * Get the entity for this context.
+ *
+ * @return This will return absent if no data is present. Otherwise the entity will be contained within the
+ * optional
+ */
+ Optional<Entity> getEntity();
+
+ /**
+ * Get the status of the entity
+ */
+ Status getStatus();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java
new file mode 100644
index 0000000..e518298
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/MvccLogEntry.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.collection;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * A Marker interface for an in flight update to allow context information to be passed between states
+ */
+public interface MvccLogEntry {
+
+
+ /**
+ * Get the stage for the current version
+ */
+ Stage getStage();
+
+ /**
+ * Get the entity to add info to the log
+ */
+ Id getEntityId();
+
+ /**
+ * Get the version of the entity
+ */
+ UUID getVersion();
+
+ /**
+ * Get the status of the entity
+ */
+ State getState();
+
+
+
+ /**
+ * The state of the entity. Is it a complete entity, a partial entity, or a deleted?
+ */
+ public enum State {
+
+ /**
+ * The logentry being written represents a complete entity
+ */
+ COMPLETE(0),
+ /**
+ * The logentry being written represents a partial entity
+ */
+ PARTIAL(1),
+
+ /**
+ * This logentry has been marked as deleted
+ */
+ DELETED(2)
+ ;
+
+ private final int id;
+
+
+ private State( final int id ) {
+ this.id = id;
+ }
+
+
+ /**
+ * Returns true if this stage is transient and should not be retained in the datastore permanently Stages such as
+ * start and write don't need to be retained, but can be used to signal "in flight" updates
+ */
+
+
+ public int getId() {
+ return this.id;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/VersionSet.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/VersionSet.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/VersionSet.java
new file mode 100644
index 0000000..8ee9cdc
--- /dev/null
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/VersionSet.java
@@ -0,0 +1,67 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.persistence.collection;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ * Represents a set of entities
+ */
+public interface VersionSet {
+
+ /**
+ * Get the entity from the result set
+ * @param entityId
+ * @return
+ */
+ public MvccLogEntry getMaxVersion( Id entityId );
+
+ /**
+ * Get the number of entities in this set
+ * @return
+ */
+ public int size();
+
+ /**
+ * Return true if the set is empty
+ * @return
+ */
+ public boolean isEmpty();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index 82e1c76..416cb9f 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -18,7 +18,7 @@
package org.apache.usergrid.persistence.collection.exception;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
public class CollectionRuntimeException extends RuntimeException {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteCommitException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteCommitException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteCommitException.java
index e09c086..b0b8b9c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteCommitException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteCommitException.java
@@ -18,8 +18,8 @@
package org.apache.usergrid.persistence.collection.exception;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+
public class WriteCommitException extends CollectionRuntimeException {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteOptimisticVerifyException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteOptimisticVerifyException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteOptimisticVerifyException.java
index a7df320..ca9c7aa 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteOptimisticVerifyException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteOptimisticVerifyException.java
@@ -18,8 +18,8 @@
package org.apache.usergrid.persistence.collection.exception;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+
public class WriteOptimisticVerifyException extends CollectionRuntimeException {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteStartException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteStartException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteStartException.java
index 9cf5cb5..f8a90df 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteStartException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteStartException.java
@@ -18,8 +18,8 @@
package org.apache.usergrid.persistence.collection.exception;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+
public class WriteStartException extends CollectionRuntimeException {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
index 7e7f05b..a20e090 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/WriteUniqueVerifyException.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.collection.exception;
import java.util.Map;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.field.Field;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
index cb0087b..8015ca9 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/guice/CollectionModule.java
@@ -28,7 +28,7 @@ import org.apache.usergrid.persistence.collection.impl.EntityCollectionManagerSy
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueSerializationStrategyImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteStart;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index 5fe2fbb..c47bf1a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@ -21,6 +21,8 @@ package org.apache.usergrid.persistence.collection.impl;
import java.util.Collection;
import java.util.Collections;
+import java.util.List;
+import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -28,17 +30,18 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.EntitySet;
+import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.collection.guice.CollectionTaskExecutor;
import org.apache.usergrid.persistence.collection.guice.Write;
import org.apache.usergrid.persistence.collection.guice.WriteUpdate;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.delete.MarkStart;
-import org.apache.usergrid.persistence.collection.mvcc.stage.load.Load;
-import org.apache.usergrid.persistence.collection.mvcc.stage.load.GetVersion;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.RollbackAction;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteCommit;
import org.apache.usergrid.persistence.collection.mvcc.stage.write.WriteOptimisticVerify;
@@ -54,7 +57,6 @@ import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
-import java.util.UUID;
import rx.Observable;
import rx.functions.Action1;
@@ -66,7 +68,6 @@ import rx.schedulers.Schedulers;
* Simple implementation. Should perform writes, delete and load.
*
* TODO: maybe refactor the stage operations into their own classes for clarity and organization?
- *
*/
public class EntityCollectionManagerImpl implements EntityCollectionManager {
@@ -83,7 +84,6 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final WriteOptimisticVerify writeOptimisticVerify;
private final WriteCommit writeCommit;
private final RollbackAction rollback;
- private final GetVersion getVersion;
//delete stages
@@ -92,8 +92,10 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
private final TaskExecutor taskExecutor;
+ private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
private final MvccEntitySerializationStrategy entitySerializationStrategy;
+
@Inject
public EntityCollectionManagerImpl( final UUIDService uuidService, @Write final WriteStart writeStart,
@WriteUpdate final WriteStart writeUpdate,
@@ -103,8 +105,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
final MarkStart markStart, final MarkCommit markCommit,
final MvccEntitySerializationStrategy entitySerializationStrategy,
@CollectionTaskExecutor final TaskExecutor taskExecutor,
- @Assisted final CollectionScope collectionScope
- ) {
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ @Assisted final CollectionScope collectionScope ) {
Preconditions.checkNotNull( uuidService, "uuidService must be defined" );
@@ -118,14 +120,15 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
this.writeCommit = writeCommit;
this.rollback = rollback;
- this.load = load;
+
this.markStart = markStart;
this.markCommit = markCommit;
- this.getVersion = getVersion;
this.uuidService = uuidService;
this.collectionScope = collectionScope;
this.taskExecutor = taskExecutor;
+ this.entitySerializationStrategy = entitySerializationStrategy;
+ this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
}
@@ -133,8 +136,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
public Observable<Entity> write( final Entity entity ) {
//do our input validation
- Preconditions.checkNotNull( entity,
- "Entity is required in the new stage of the mvcc write" );
+ Preconditions.checkNotNull( entity, "Entity is required in the new stage of the mvcc write" );
final Id entityId = entity.getId();
@@ -144,7 +146,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
// create our observable and start the write
final CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData,writeStart );
+ Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeStart );
// execute all validation stages concurrently. Needs refactored when this is done.
// https://github.com/Netflix/RxJava/issues/627
@@ -173,14 +175,13 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
- return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) )
- .map( markStart ).doOnNext( markCommit ).map( new Func1<CollectionIoEvent<MvccEntity>,
- Void>() {
- @Override
- public Void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
- return null;
- }
- } );
+ return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) ).map( markStart )
+ .doOnNext( markCommit ).map( new Func1<CollectionIoEvent<MvccEntity>, Void>() {
+ @Override
+ public Void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
+ return null;
+ }
+ } );
}
@@ -191,12 +192,12 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityId.getUuid(), "Entity id uuid required in load stage" );
Preconditions.checkNotNull( entityId.getType(), "Entity id type required in load stage" );
- return load( Collections.singleton(entityId) ).map( new Func1<EntitySet, Entity>() {
+ return load( Collections.singleton( entityId ) ).map( new Func1<EntitySet, Entity>() {
@Override
public Entity call( final EntitySet entitySet ) {
final MvccEntity entity = entitySet.getEntity( entityId );
- if(entity == null){
+ if ( entity == null ) {
return null;
}
@@ -211,8 +212,8 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
Preconditions.checkNotNull( entityIds, "entityIds cannot be null" );
- final EntitySet
- results = entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
+ final EntitySet results =
+ entitySerializationStrategy.load( collectionScope, entityIds, UUIDGenerator.newTimeUUID() );
return Observable.just( results );
}
@@ -235,7 +236,7 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
CollectionIoEvent<Entity> writeData = new CollectionIoEvent<Entity>( collectionScope, entity );
- Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData,writeUpdate );
+ Observable<CollectionIoEvent<MvccEntity>> observable = stageRunner( writeData, writeUpdate );
return observable.map( writeCommit ).doOnNext( new Action1<Entity>() {
@@ -243,49 +244,53 @@ public class EntityCollectionManagerImpl implements EntityCollectionManager {
public void call( final Entity entity ) {
log.debug( "sending entity to the queue" );
- //we an update, signal the fix
+ //we an update, signal the fix
//TODO T.N Change this to fire a task
-// Observable.from( new CollectionIoEvent<Id>(collectionScope, entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
+ // Observable.from( new CollectionIoEvent<Id>(collectionScope,
+ // entityId ) ).map( load ).subscribeOn( Schedulers.io() ).subscribe();
}
} ).doOnError( rollback );
}
+
// fire the stages
- public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData, WriteStart writeState ) {
+ public Observable<CollectionIoEvent<MvccEntity>> stageRunner( CollectionIoEvent<Entity> writeData,
+ WriteStart writeState ) {
return Observable.from( writeData ).map( writeState ).doOnNext( new Action1<CollectionIoEvent<MvccEntity>>() {
- @Override
- public void call(
- final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
-
- Observable<CollectionIoEvent<MvccEntity>> unique =
- Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
- .doOnNext( writeVerifyUnique );
-
+ @Override
+ public void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
- // optimistic verification
- Observable<CollectionIoEvent<MvccEntity>> optimistic =
- Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
- .doOnNext( writeOptimisticVerify );
+ Observable<CollectionIoEvent<MvccEntity>> unique =
+ Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ .doOnNext( writeVerifyUnique );
- //wait for both to finish
- Observable.merge( unique, optimistic ).toBlocking().last();
+ // optimistic verification
+ Observable<CollectionIoEvent<MvccEntity>> optimistic =
+ Observable.from( mvccEntityCollectionIoEvent ).subscribeOn( Schedulers.io() )
+ .doOnNext( writeOptimisticVerify );
- }
- } );
+ //wait for both to finish
+ Observable.merge( unique, optimistic ).toBlocking().last();
+ }
+ } );
}
@Override
- public Observable<UUID> getLatestVersion(Id entityId) {
- return Observable.from(
- new CollectionIoEvent<Id>( collectionScope, entityId ) ).map(getVersion);
- }
+ public Observable<VersionSet> getLatestVersion(
+ Collection<Id> entityIds ) {
+
+ VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
+ UUIDGenerator.newTimeUUID() );
+
+ return Observable.just(logEntries);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 2d30d36..422a155 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@ -12,7 +12,7 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
index 0467c95..b9277eb 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccEntitySerializationStrategy.java
@@ -21,12 +21,11 @@ package org.apache.usergrid.persistence.collection.mvcc;
import java.util.Collection;
import java.util.Iterator;
-import java.util.List;
import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Id;
import com.netflix.astyanax.MutationBatch;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
index 2d218c1..927a60c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/MvccLogEntrySerializationStrategy.java
@@ -19,15 +19,16 @@
package org.apache.usergrid.persistence.collection.mvcc;
+import java.util.Collection;
import java.util.List;
import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.VersionSet;
import org.apache.usergrid.persistence.model.entity.Id;
import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
/**
@@ -48,13 +49,12 @@ public interface MvccLogEntrySerializationStrategy {
* Load and return the stage with the given id and a version that is <= the version provided
*
* @param context The context to persist the entity into
- * @param entityId The entity id to load
- * @param version The version to load. This will return the version == the given version
+ * @param entityIds The entity id to load
+ * @param version The max version to load. This will return the first version <= the given version
*
- * @return The deserialized version of the log entry. Null if no version == the current version exists
+ * @return The deserialized version of the log entry
*/
- public MvccLogEntry load( final CollectionScope context, final Id entityId, final UUID version )
- throws ConnectionException;
+ public VersionSet load( final CollectionScope context, final Collection<Id> entityIds, final UUID version );
/**
* Load a list, from highest to lowest of the stage with versions <= version up to maxSize elements
@@ -66,8 +66,7 @@ public interface MvccLogEntrySerializationStrategy {
*
* @return A list of entities up to max size ordered from max(UUID)=> min(UUID)
*/
- public List<MvccLogEntry> load( CollectionScope context, Id entityId, UUID version, int maxSize )
- throws ConnectionException;
+ public List<MvccLogEntry> load( CollectionScope context, Id entityId, UUID version, int maxSize );
/**
* MarkCommit the stage from the context with the given entityId and version
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
index 68cbd58..8171f10 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGenerator.java
@@ -19,11 +19,8 @@ package org.apache.usergrid.persistence.collection.mvcc.changelog;
import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.UUID;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
/**
* This change log generator takes one or more entity versions and generates the change-log.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
index 014d249..eb4d019 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/changelog/ChangeLogGeneratorImpl.java
@@ -24,7 +24,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Set;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.field.Field;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccEntity.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccEntity.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccEntity.java
deleted file mode 100644
index c9fb99f..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccEntity.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.collection.mvcc.entity;
-
-
-import org.apache.usergrid.persistence.core.entity.EntityVersion;
-import org.apache.usergrid.persistence.model.entity.Entity;
-
-import com.google.common.base.Optional;
-
-
-/**
- * An entity with internal information for versioning
- */
-public interface MvccEntity extends EntityVersion{
-
- /**
- * The possible State of the mvccEntity
- */
- public enum Status {
-
- /**
- * The entity being written represents a complete entity
- */
- COMPLETE,
-
- /**
- * The entity being written represents a partial entity
- */
- PARTIAL,
-
- /**
- * This entity has been marked as deleted
- */
- DELETED
- ;
- }
-
-
- /**
- * Get the entity for this context.
- *
- * @return This will return absent if no data is present. Otherwise the entity will be contained within the
- * optional
- */
- Optional<Entity> getEntity();
-
- /**
- * Get the status of the entity
- */
- Status getStatus();
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccLogEntry.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccLogEntry.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccLogEntry.java
deleted file mode 100644
index b45c1fb..0000000
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccLogEntry.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.collection.mvcc.entity;
-
-
-import java.util.UUID;
-
-import org.apache.usergrid.persistence.model.entity.Id;
-
-
-/**
- * A Marker interface for an in flight update to allow context information to be passed between states
- */
-public interface MvccLogEntry {
-
-
- /**
- * Get the stage for the current version
- */
- Stage getStage();
-
- /**
- * Get the entity to add info to the log
- */
- Id getEntityId();
-
- /**
- * Get the version of the entity
- */
- UUID getVersion();
-
- /**
- * Get the status of the entity
- */
- State getState();
-
-
-
- /**
- * The state of the entity. Is it a complete entity, a partial entity, or a deleted?
- */
- public enum State {
-
- /**
- * The logentry being written represents a complete entity
- */
- COMPLETE(0),
- /**
- * The logentry being written represents a partial entity
- */
- PARTIAL(1),
-
- /**
- * This logentry has been marked as deleted
- */
- DELETED(2)
- ;
-
- private final int id;
-
-
- private State( final int id ) {
- this.id = id;
- }
-
-
- /**
- * Returns true if this stage is transient and should not be retained in the datastore permanently Stages such as
- * start and write don't need to be retained, but can be used to signal "in flight" updates
- */
-
-
- public int getId() {
- return this.id;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccValidationUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccValidationUtils.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccValidationUtils.java
index f40572a..1f0c91b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccValidationUtils.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/MvccValidationUtils.java
@@ -20,6 +20,7 @@ package org.apache.usergrid.persistence.collection.mvcc.entity;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityDeleteEvent.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityDeleteEvent.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityDeleteEvent.java
index 561b345..1f8fc24 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityDeleteEvent.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityDeleteEvent.java
@@ -22,7 +22,7 @@ package org.apache.usergrid.persistence.collection.mvcc.entity.impl;
import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityEvent.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityEvent.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityEvent.java
index 3b18e5b..f9f1389 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityEvent.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityEvent.java
@@ -19,7 +19,7 @@
package org.apache.usergrid.persistence.collection.mvcc.entity.impl;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import java.io.Serializable;
import java.util.UUID;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
index 8f29ac0..399d4f6 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityImpl.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.collection.mvcc.entity.impl;
import java.util.UUID;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityWriteEvent.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityWriteEvent.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityWriteEvent.java
index 1b7bae1..84cfd18 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityWriteEvent.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccEntityWriteEvent.java
@@ -22,8 +22,7 @@ package org.apache.usergrid.persistence.collection.mvcc.entity.impl;
import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.collection.MvccEntity;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImpl.java
index a50eeac..3d59ebd 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/entity/impl/MvccLogEntryImpl.java
@@ -21,7 +21,7 @@ package org.apache.usergrid.persistence.collection.mvcc.entity.impl;
import java.util.UUID;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/EntityUpdateEvent.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/EntityUpdateEvent.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/EntityUpdateEvent.java
index c70245a..3951226 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/EntityUpdateEvent.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/EntityUpdateEvent.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.persistence.collection.mvcc.stage;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index 1e01c02..9e2d52c 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@ -29,8 +29,8 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
index 2529997..c80b076 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkStart.java
@@ -27,8 +27,8 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
index 8561708..e1becf8 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/RollbackAction.java
@@ -24,7 +24,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
index 82ef74c..ba638a5 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteCommit.java
@@ -27,8 +27,8 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.exception.WriteCommitException;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerify.java
index dcdb408..b031237 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteOptimisticVerify.java
@@ -24,11 +24,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
@@ -36,23 +35,21 @@ import org.apache.usergrid.persistence.model.entity.Entity;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.functions.Action1;
-import rx.functions.Func1;
/**
* This phase should execute any optimistic verification on the MvccEntity
*/
@Singleton
-public class WriteOptimisticVerify
- implements Action1<CollectionIoEvent<MvccEntity>> {
+public class WriteOptimisticVerify implements Action1<CollectionIoEvent<MvccEntity>> {
private static final Logger log = LoggerFactory.getLogger( WriteOptimisticVerify.class );
private final MvccLogEntrySerializationStrategy logEntryStrat;
+
@Inject
public WriteOptimisticVerify( MvccLogEntrySerializationStrategy logEntryStrat ) {
this.logEntryStrat = logEntryStrat;
@@ -74,32 +71,21 @@ public class WriteOptimisticVerify
CollectionScope collectionScope = ioevent.getEntityCollection();
- if(entity.getVersion() == null){
+ if ( entity.getVersion() == null ) {
return;
}
- try {
- List<MvccLogEntry> versions = logEntryStrat.load(
- collectionScope, entity.getId(), entity.getVersion(), 2 );
-
- // Previous log entry must be committed, otherwise somebody is already writing
- if ( versions.size() > 1
- && versions.get(1).getStage().ordinal() < Stage.COMMITTED.ordinal() ) {
-
- log.debug("Conflict writing entity id {} version {}",
- entity.getId().toString(), entity.getVersion().toString());
-
- throw new WriteOptimisticVerifyException( mvccEntity, collectionScope,
- "Change conflict, not first writer");
- }
-
- } catch ( ConnectionException e ) {
- log.error( "Error reading entity log", e );
- throw new CollectionRuntimeException( mvccEntity, collectionScope,
- "Error reading entity log", e );
- }
+ List<MvccLogEntry> versions = logEntryStrat.load( collectionScope, entity.getId(), entity.getVersion(), 2 );
- }
+ // Previous log entry must be committed, otherwise somebody is already writing
+ if ( versions.size() > 1 && versions.get( 1 ).getStage().ordinal() < Stage.COMMITTED.ordinal() ) {
+ log.debug( "Conflict writing entity id {} version {}", entity.getId().toString(),
+ entity.getVersion().toString() );
+
+ throw new WriteOptimisticVerifyException( mvccEntity, collectionScope,
+ "Change conflict, not first writer" );
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
index e2bad71..0056f46 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteStart.java
@@ -9,13 +9,12 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.exception.WriteStartException;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.mvcc.entity.Stage;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccLogEntryImpl;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
-import org.apache.usergrid.persistence.collection.util.EntityUtils;
import org.apache.usergrid.persistence.model.entity.Entity;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
index 6ca3445..162bcac 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/write/WriteUniqueVerify.java
@@ -30,7 +30,7 @@ import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
import org.apache.usergrid.persistence.collection.mvcc.stage.CollectionIoEvent;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java
index c4fddd2..b2be289 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/EntityRepair.java
@@ -19,7 +19,7 @@ package org.apache.usergrid.persistence.collection.serialization;/*
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/OptimisticUpdate.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/OptimisticUpdate.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/OptimisticUpdate.java
index 36b9527..40bde16 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/OptimisticUpdate.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/OptimisticUpdate.java
@@ -3,7 +3,7 @@ package org.apache.usergrid.persistence.collection.serialization;
import java.util.UUID;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
index 62f4915..a94c688 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityRepairImpl.java
@@ -28,7 +28,7 @@ import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationSt
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLog;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGenerator;
import org.apache.usergrid.persistence.collection.mvcc.changelog.ChangeLogGeneratorImpl;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.model.entity.Entity;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
index 9e03754..921093b 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntitySetImpl.java
@@ -22,8 +22,7 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.usergrid.persistence.collection.EntitySet;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
-import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
index d87f850..2b99033 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/LogEntryIterator.java
@@ -8,7 +8,7 @@ import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
+import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7af7e702/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
index 5449e46..8f51cbc 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/MvccEntitySerializationStrategyImpl.java
@@ -38,7 +38,7 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.EntityRepair;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
@@ -63,7 +63,6 @@ import com.netflix.astyanax.ColumnListMutation;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
-import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.ColumnList;
import com.netflix.astyanax.model.CompositeBuilder;
[05/16] git commit: Initial refactor of results loading to use
multiget. Need to push load logic down to impelmentations.
Posted by to...@apache.org.
Initial refactor of results loading to use multiget. Need to push load logic down to impelmentations.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/63094abc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/63094abc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/63094abc
Branch: refs/heads/two-dot-o
Commit: 63094abc3d52fb6139d8a0828328d0bc4c111e0c
Parents: ac61f13
Author: Todd Nine <to...@apache.org>
Authored: Thu Oct 9 13:23:35 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Thu Oct 9 13:23:35 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityDeleteListener.java | 2 +-
.../corepersistence/CpRelationManager.java | 6 +-
.../results/AbstractIdLoader.java | 130 +++++++++++++++++++
.../corepersistence/results/EntitiesLoader.java | 51 ++++++++
.../corepersistence/results/IdsLoader.java | 46 +++++++
.../corepersistence/results/RefsLoader.java | 46 +++++++
.../corepersistence/results/ResultsLoader.java | 57 ++++++++
.../results/ResultsLoaderFactory.java | 53 ++++++++
.../results/ResultsLoaderFactoryImpl.java | 70 ++++++++++
.../CpEntityDeleteListenerTest.java | 2 +-
.../CpEntityIndexDeleteListenerTest.java | 3 +-
11 files changed, 461 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
index 6b6fa59..70df7d5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityDeleteListener.java
@@ -26,7 +26,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityDeleteEvent;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.entity.EntityVersion;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index bcfe215..03aada3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -1551,8 +1551,12 @@ public class CpRelationManager implements RelationManager {
*/
private Results buildResults(Query query, CandidateResults crs, String collName ) {
+
logger.debug("buildResults() for {} from {} candidates", collName, crs.size());
+
+ //TODO T.N Change to results loader here
+
Results results = null;
EntityIndex index = managerCache.getEntityIndex(applicationScope);
@@ -1574,7 +1578,7 @@ public class CpRelationManager implements RelationManager {
EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
- UUID latestVersion = ecm.getLatestVersion( cr.getId() ).toBlocking().lastOrDefault(null);
+ UUID latestVersion = ecm.getLatestVersion( Collections.singleton( cr.getId()) ).toBlocking().last().getMaxVersion( cr.getId() ).getVersion();
if ( logger.isDebugEnabled() ) {
logger.debug("Getting version for entity {} from scope\n app {}\n owner {}\n name {}",
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java
new file mode 100644
index 0000000..b034a53
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/AbstractIdLoader.java
@@ -0,0 +1,130 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.apache.usergrid.corepersistence.CpNamingUtils;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+public abstract class AbstractIdLoader implements ResultsLoader{
+
+ @Override
+ public Results getResults( final ApplicationScope applicationScope, final CandidateResults crs ) {
+// Map<Id, CandidateResult> latestVersions = new LinkedHashMap<Id, CandidateResult>();
+//
+// Iterator<CandidateResult> iter = crs.iterator();
+// while ( iter.hasNext() ) {
+//
+// CandidateResult cr = iter.next();
+//
+// CollectionScope collScope = new CollectionScopeImpl(
+// applicationScope.getApplication(),
+// applicationScope.getApplication(),
+// CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
+//
+// EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collScope);
+//
+// UUID latestVersion = ecm.getLatestVersion( cr.getId() ).toBlocking().lastOrDefault(null);
+//
+// if ( logger.isDebugEnabled() ) {
+// logger.debug("Getting version for entity {} from scope\n app {}\n owner {}\n name {}",
+// new Object[] {
+// cr.getId(),
+// collScope.getApplication(),
+// collScope.getOwner(),
+// collScope.getName()
+// });
+// }
+//
+// if ( latestVersion == null ) {
+// logger.error("Version for Entity {}:{} not found",
+// cr.getId().getType(), cr.getId().getUuid());
+// continue;
+// }
+//
+// if ( cr.getVersion().compareTo( latestVersion) < 0 ) {
+// logger.debug("Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+// new Object[] { cr.getId().getUuid(), cr.getId().getType(),
+// cr.getVersion(), latestVersion});
+//
+// IndexScope indexScope = new IndexScopeImpl(
+// cpHeadEntity.getId(),
+// CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
+// indexBatch.deindex( indexScope, cr);
+//
+// continue;
+// }
+//
+// CandidateResult alreadySeen = latestVersions.get( cr.getId() );
+//
+// if ( alreadySeen == null ) { // never seen it, so add to map
+// latestVersions.put( cr.getId(), cr );
+//
+// } else {
+// // we seen this id before, only add entity if we now have newer version
+// if ( latestVersion.compareTo( alreadySeen.getVersion() ) > 0 ) {
+//
+// latestVersions.put( cr.getId(), cr);
+//
+// IndexScope indexScope = new IndexScopeImpl(
+// cpHeadEntity.getId(),
+// CpNamingUtils.getCollectionScopeNameFromEntityType( cr.getId().getType() ));
+// indexBatch.deindex( indexScope, alreadySeen);
+// }
+// }
+// }
+//
+// indexBatch.execute();
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java
new file mode 100644
index 0000000..d20f677
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntitiesLoader.java
@@ -0,0 +1,51 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+
+
+public class EntitiesLoader implements ResultsLoader {
+ @Override
+ public Results getResults( final ApplicationScope scope, final CandidateResults crs ) {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java
new file mode 100644
index 0000000..7796028
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/IdsLoader.java
@@ -0,0 +1,46 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+
+
+public class IdsLoader extends AbstractIdLoader{
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java
new file mode 100644
index 0000000..ad852d5
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/RefsLoader.java
@@ -0,0 +1,46 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+
+
+public class RefsLoader extends AbstractIdLoader{
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
new file mode 100644
index 0000000..8a17c6e
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
@@ -0,0 +1,57 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.query.CandidateResults;
+
+
+/**
+ * Interface for loading results
+ */
+public interface ResultsLoader {
+
+ /**
+ * Using the candidate results, get the results
+ * @param crs
+ * @return
+ */
+ public Results getResults(final ApplicationScope scope, final CandidateResults crs);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
new file mode 100644
index 0000000..5c5892c
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
@@ -0,0 +1,53 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+/**
+ * Factory for creating results
+ */
+
+public interface ResultsLoaderFactory {
+
+
+ /**
+ * Get the load for results
+ * @return
+ */
+ public ResultsLoader getLoader();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
new file mode 100644
index 0000000..edaca78
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
@@ -0,0 +1,70 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.corepersistence.results;/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * Factory for creating results
+ */
+@Singleton
+public class ResultsLoaderFactoryImpl implements ResultsLoaderFactory {
+
+
+ private final EntitiesLoader entityLoader;
+ private final IdsLoader idsLoader;
+ private final RefsLoader refsLoader;
+
+
+ @Inject
+ public ResultsLoaderFactoryImpl( final EntitiesLoader entityLoader, final IdsLoader idsLoader,
+ final RefsLoader refsLoader ) {
+ this.entityLoader = entityLoader;
+ this.idsLoader = idsLoader;
+ this.refsLoader = refsLoader;
+ }
+
+
+ @Override
+ public ResultsLoader getLoader() {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
index dfea003..16d2e79 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityDeleteListenerTest.java
@@ -28,7 +28,7 @@ import org.junit.runner.RunWith;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityDeleteEvent;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
import org.apache.usergrid.persistence.core.entity.EntityVersion;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/63094abc/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
index 77c62c5..6b92d90 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListenerTest.java
@@ -29,9 +29,8 @@ import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
-import org.apache.usergrid.Application;
import org.apache.usergrid.persistence.collection.CollectionScope;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityDeleteEvent;
import org.apache.usergrid.persistence.collection.mvcc.entity.impl.MvccEntityImpl;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;