You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2014/10/10 17:50:50 UTC
[43/43] git commit: merge from two-o
merge from two-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d9ace9e0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d9ace9e0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d9ace9e0
Branch: refs/heads/two-dot-o-events
Commit: d9ace9e0916e69e6b4d9d11a9c062219a5ed73c0
Parents: 5a7aa5b befcdca
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Oct 10 09:49:13 2014 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Oct 10 09:49:13 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityDeleteListener.java | 2 +-
.../corepersistence/CpEntityManager.java | 392 ++++++++-----------
.../corepersistence/CpEntityManagerFactory.java | 217 +++++++---
.../corepersistence/CpEntityMapUtils.java | 323 ---------------
.../corepersistence/CpManagerCache.java | 2 +-
.../usergrid/corepersistence/CpNamingUtils.java | 106 -----
.../corepersistence/CpRelationManager.java | 151 ++-----
.../usergrid/corepersistence/CpVisitor.java | 32 ++
.../usergrid/corepersistence/CpWalker.java | 238 +++++++++++
.../HybridEntityManagerFactory.java | 10 +
.../corepersistence/results/EntityVerifier.java | 126 ++++++
.../results/FilteringLoader.java | 235 +++++++++++
.../corepersistence/results/IdsVerifier.java | 46 +++
.../corepersistence/results/RefsVerifier.java | 42 ++
.../corepersistence/results/ResultsLoader.java | 38 ++
.../results/ResultsLoaderFactory.java | 39 ++
.../results/ResultsLoaderFactoryImpl.java | 67 ++++
.../results/ResultsVerifier.java | 54 +++
.../results/VersionVerifier.java | 82 ++++
.../corepersistence/util/CpEntityMapUtils.java | 323 +++++++++++++++
.../corepersistence/util/CpNamingUtils.java | 106 +++++
.../event/EntityVersionDeletedImpl.java | 2 +-
.../usergrid/persistence/EntityManager.java | 10 +
.../persistence/EntityManagerFactory.java | 4 +
.../apache/usergrid/persistence/Results.java | 6 +-
.../cassandra/EntityManagerFactoryImpl.java | 10 +
.../cassandra/EntityManagerImpl.java | 17 +
.../org/apache/usergrid/CoreApplication.java | 1 +
.../CpEntityDeleteListenerTest.java | 2 +-
.../CpEntityIndexDeleteListenerTest.java | 3 +-
.../corepersistence/CpEntityMapUtilsTest.java | 2 +
.../corepersistence/StaleIndexCleanupTest.java | 247 ++++++++++++
.../PerformanceEntityRepersistTest.java | 240 ++++++++++++
.../collection/EntityCollectionManager.java | 22 +-
.../persistence/collection/EntitySet.java | 47 +++
.../persistence/collection/MvccEntity.java | 68 ++++
.../persistence/collection/MvccLogEntry.java | 94 +++++
.../persistence/collection/VersionSet.java | 67 ++++
.../collection/event/EntityVersionDeleted.java | 2 +-
.../exception/CollectionRuntimeException.java | 2 +-
.../exception/WriteCommitException.java | 4 +-
.../WriteOptimisticVerifyException.java | 4 +-
.../exception/WriteStartException.java | 4 +-
.../exception/WriteUniqueVerifyException.java | 3 +-
.../collection/guice/CollectionModule.java | 14 +-
.../impl/EntityCollectionManagerImpl.java | 214 +++++++---
.../impl/EntityVersionCleanupTask.java | 9 +-
.../mvcc/MvccEntitySerializationStrategy.java | 20 +-
.../mvcc/MvccLogEntrySerializationStrategy.java | 17 +-
.../collection/mvcc/changelog/ChangeLog.java | 68 ++++
.../mvcc/changelog/ChangeLogEntry.java | 104 -----
.../mvcc/changelog/ChangeLogGenerator.java | 16 +-
.../mvcc/changelog/ChangeLogGeneratorImpl.java | 138 +++----
.../mvcc/changelog/ChangeLogImpl.java | 135 +++++++
.../collection/mvcc/entity/MvccEntity.java | 68 ----
.../collection/mvcc/entity/MvccLogEntry.java | 93 -----
.../mvcc/entity/MvccValidationUtils.java | 4 +
.../mvcc/entity/impl/MvccEntityDeleteEvent.java | 2 +-
.../mvcc/entity/impl/MvccEntityEvent.java | 2 +-
.../mvcc/entity/impl/MvccEntityImpl.java | 2 +-
.../mvcc/entity/impl/MvccEntityWriteEvent.java | 3 +-
.../mvcc/entity/impl/MvccLogEntryImpl.java | 2 +-
.../mvcc/stage/EntityUpdateEvent.java | 1 -
.../mvcc/stage/delete/MarkCommit.java | 71 +++-
.../collection/mvcc/stage/delete/MarkStart.java | 4 +-
.../collection/mvcc/stage/load/Load.java | 98 -----
.../mvcc/stage/write/RollbackAction.java | 6 +-
.../mvcc/stage/write/WriteCommit.java | 13 +-
.../mvcc/stage/write/WriteOptimisticVerify.java | 44 +--
.../collection/mvcc/stage/write/WriteStart.java | 12 +-
.../mvcc/stage/write/WriteUniqueVerify.java | 21 +-
.../collection/serialization/EntityRepair.java | 38 ++
.../serialization/OptimisticUpdate.java | 2 +-
.../serialization/SerializationFig.java | 19 +
.../collection/serialization/UniqueValue.java | 6 -
.../UniqueValueSerializationStrategy.java | 6 +-
.../serialization/impl/EntityRepairImpl.java | 149 +++++++
.../serialization/impl/EntitySetImpl.java | 62 +++
.../impl/EntityVersionSerializer.java | 13 +-
.../serialization/impl/FieldSerializer.java | 86 ++--
.../serialization/impl/LogEntryIterator.java | 2 +-
.../MvccEntitySerializationStrategyImpl.java | 175 ++++++---
.../MvccLogEntrySerializationStrategyImpl.java | 129 ++++--
.../serialization/impl/UniqueValueImpl.java | 21 +-
.../UniqueValueSerializationStrategyImpl.java | 42 +-
.../serialization/impl/VersionSetImpl.java | 80 ++++
.../persistence/collection/util/RepairUtil.java | 138 -------
.../collection/EntityCollectionManagerIT.java | 322 ++++++++++++---
.../impl/EntityVersionCleanupTaskTest.java | 2 +-
.../changelog/ChangeLogGeneratorImplTest.java | 375 ++++++++++--------
.../mvcc/entity/impl/MvccEntityImplTest.java | 2 +-
.../mvcc/entity/impl/MvccLogEntryImplTest.java | 2 +-
.../mvcc/stage/AbstractMvccEntityStageTest.java | 2 +-
.../mvcc/stage/TestEntityGenerator.java | 2 +-
.../mvcc/stage/delete/MarkCommitTest.java | 6 +-
.../mvcc/stage/delete/MarkStartTest.java | 4 +-
.../collection/mvcc/stage/load/LoadTest.java | 361 -----------------
...niqueValueSerializationStrategyImplTest.java | 14 +-
.../mvcc/stage/write/WriteCommitTest.java | 4 +-
.../stage/write/WriteOptimisticVerifyTest.java | 18 +-
.../mvcc/stage/write/WriteStartTest.java | 4 +-
.../mvcc/stage/write/WriteUniqueVerifyTest.java | 12 +-
.../serialization/EntityRepairImplTest.java | 147 +++++++
.../impl/LogEntryIteratorTest.java | 2 +-
...MvccEntitySerializationStrategyImplTest.java | 40 +-
.../impl/MvccLESSTransientTest.java | 7 +-
...ccLogEntrySerializationStrategyImplTest.java | 19 +-
.../util/InvalidMvccEntityGenerator.java | 2 +-
.../util/InvalidValueGeneratorTest.java | 2 +-
.../collection/util/LogEntryMock.java | 4 +-
.../persistence/model/field/AbstractField.java | 4 +
.../persistence/model/field/ArrayField.java | 3 +
.../persistence/model/field/BooleanField.java | 6 +
.../persistence/model/field/ByteArrayField.java | 6 +
.../persistence/model/field/DoubleField.java | 6 +
.../model/field/EntityObjectField.java | 6 +
.../usergrid/persistence/model/field/Field.java | 8 +
.../persistence/model/field/FieldTypeName.java | 42 ++
.../persistence/model/field/FloatField.java | 6 +
.../persistence/model/field/IntegerField.java | 6 +
.../persistence/model/field/ListField.java | 7 +
.../persistence/model/field/LocationField.java | 6 +
.../persistence/model/field/LongField.java | 6 +
.../persistence/model/field/SetField.java | 6 +
.../persistence/model/field/StringField.java | 6 +
.../persistence/model/field/UUIDField.java | 6 +
stack/corepersistence/pom.xml | 4 +-
.../usergrid/persistence/index/EntityIndex.java | 6 +
.../index/impl/EsEntityIndexBatchImpl.java | 119 ++----
.../index/impl/EsEntityIndexImpl.java | 280 ++++++-------
.../persistence/index/impl/IndexingUtils.java | 66 ++++
.../index/query/CandidateResult.java | 30 ++
.../persistence/index/impl/EntityIndexTest.java | 5 +
.../cassandra/ManagementServiceImpl.java | 38 +-
.../org/apache/usergrid/tools/RepersistAll.java | 98 +++++
135 files changed, 5101 insertions(+), 2669 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/core/src/main/java/org/apache/usergrid/event/EntityVersionDeletedImpl.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/event/EntityVersionDeletedImpl.java
index e48fa7c,0000000..a76e1c4
mode 100644,000000..100644
--- a/stack/core/src/main/java/org/apache/usergrid/event/EntityVersionDeletedImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/event/EntityVersionDeletedImpl.java
@@@ -1,71 -1,0 +1,71 @@@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+
+package org.apache.usergrid.event;
+
+import org.apache.usergrid.persistence.collection.CollectionScope;
++import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
- import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import rx.functions.Func1;
+import rx.schedulers.Schedulers;
+
+import java.util.List;
+
+/**
+ * Purge old entity versions
+ */
+public class EntityVersionDeletedImpl implements EntityVersionDeleted{
+
+
+ private final EntityIndexBatch entityIndexBatch;
+ private final SerializationFig serializationFig;
+
+ public EntityVersionDeletedImpl(EntityIndexBatch entityIndexBatch, SerializationFig serializationFig){
+ this.entityIndexBatch = entityIndexBatch;
+ this.serializationFig = serializationFig;
+ }
+
+ @Override
+ public void versionDeleted(final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
+ final IndexScope indexScope = new IndexScopeImpl(
+ new SimpleId(scope.getOwner().getUuid(),scope.getOwner().getType()),
+ scope.getName()
+ );
+ rx.Observable.from(entityVersions)
+ .subscribeOn(Schedulers.io())
+ .buffer(serializationFig.getBufferSize())
+ .map(new Func1<List<MvccEntity>,List<MvccEntity>>() {
+ @Override
+ public List<MvccEntity> call(List<MvccEntity> entityList) {
+ for(MvccEntity entity : entityList){
+ entityIndexBatch.deindex(indexScope,entityId,entity.getVersion());
+ }
+ entityIndexBatch.execute();
+ return entityList;
+ }
+ }).toBlocking().last();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
index cfe0596,aedcf3a..c4b4408
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/event/EntityVersionDeleted.java
@@@ -22,7 -22,6 +22,7 @@@ import java.util.List
import java.util.UUID;
import org.apache.usergrid.persistence.collection.CollectionScope;
- import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
++import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.model.entity.Id;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
index f255beb,c57f666..7b96d38
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityCollectionManagerImpl.java
@@@ -19,7 -19,18 +19,19 @@@
package org.apache.usergrid.persistence.collection.impl;
+import org.apache.usergrid.persistence.collection.EntityVersionCleanupFactory;
+ import java.net.ConnectException;
+ import java.util.*;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.List;
+
+ import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+ import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+ import org.apache.usergrid.persistence.collection.serialization.UniqueValueSet;
+ import org.apache.usergrid.persistence.model.entity.SimpleId;
+ import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -87,23 -98,32 +99,33 @@@ public class EntityCollectionManagerImp
private final MarkCommit markCommit;
private final TaskExecutor taskExecutor;
-
+ private EntityVersionCleanupFactory entityVersionCleanupFactory;
+ private final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy;
+ private final MvccEntitySerializationStrategy entitySerializationStrategy;
+ private UniqueValueSerializationStrategy uniqueValueSerializationStrategy;
+
@Inject
- public EntityCollectionManagerImpl(final UUIDService uuidService, @Write final WriteStart writeStart,
- @WriteUpdate final WriteStart writeUpdate,
- final WriteUniqueVerify writeVerifyUnique,
- final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback,
- final MarkStart markStart, final MarkCommit markCommit,
+ public EntityCollectionManagerImpl( final UUIDService uuidService, @Write final WriteStart writeStart,
+ @WriteUpdate final WriteStart writeUpdate,
+ final WriteUniqueVerify writeVerifyUnique,
+ final WriteOptimisticVerify writeOptimisticVerify,
- final WriteCommit writeCommit, final RollbackAction rollback, final Load load,
++ final WriteCommit writeCommit, final RollbackAction rollback,
+ final MarkStart markStart, final MarkCommit markCommit,
- @CollectionTaskExecutor
- final TaskExecutor taskExecutor,
+ final EntityVersionCleanupFactory entityVersionCleanupFactory,
- @Assisted final CollectionScope collectionScope) {
+ final MvccEntitySerializationStrategy entitySerializationStrategy,
+ final UniqueValueSerializationStrategy uniqueValueSerializationStrategy,
+ final MvccLogEntrySerializationStrategy mvccLogEntrySerializationStrategy,
+ @CollectionTaskExecutor final TaskExecutor taskExecutor,
+ @Assisted final CollectionScope collectionScope
+ ) {
+ this.uniqueValueSerializationStrategy = uniqueValueSerializationStrategy;
+ this.entitySerializationStrategy = entitySerializationStrategy;
+
- Preconditions.checkNotNull( uuidService, "uuidService must be defined" );
+ Preconditions.checkNotNull(uuidService, "uuidService must be defined");
- MvccValidationUtils.validateCollectionScope( collectionScope );
+ MvccValidationUtils.validateCollectionScope(collectionScope);
this.writeStart = writeStart;
this.writeUpdate = writeUpdate;
@@@ -119,7 -139,7 +141,8 @@@
this.uuidService = uuidService;
this.collectionScope = collectionScope;
this.taskExecutor = taskExecutor;
+ this.entityVersionCleanupFactory = entityVersionCleanupFactory;
+ this.mvccLogEntrySerializationStrategy = mvccLogEntrySerializationStrategy;
}
@@@ -145,14 -164,14 +167,14 @@@
// observable = Concurrent.concurrent( observable, Schedulers.io(), new WaitZip(),
// writeVerifyUnique, writeOptimisticVerify );
- observable.map(writeCommit).doOnNext( new Action1<Entity>() {
+ observable.map(writeCommit).doOnNext(new Action1<Entity>() {
@Override
- public void call( final Entity entity ) {
+ public void call(final Entity entity) {
//TODO fire a task here
-
+ taskExecutor.submit(entityVersionCleanupFactory.getTask(entityId,entity.getVersion()));
//post-processing to come later. leave it empty for now.
}
- } ).doOnError( rollback );
+ }).doOnError(rollback);
// return the commit result.
@@@ -161,20 -180,19 +183,20 @@@
@Override
- public Observable<Void> delete( final Id entityId ) {
+ public Observable<Void> delete(final Id entityId) {
+
+ Preconditions.checkNotNull(entityId, "Entity id is required in this stage");
+ Preconditions.checkNotNull(entityId.getUuid(), "Entity id is required in this stage");
+ Preconditions.checkNotNull(entityId.getType(), "Entity type is required in this stage");
- Preconditions.checkNotNull( entityId, "Entity id is required in this stage" );
- Preconditions.checkNotNull( entityId.getUuid(), "Entity id is required in this stage" );
- Preconditions.checkNotNull( entityId.getType(), "Entity type is required in this stage" );
+
- return Observable.from( new CollectionIoEvent<Id>( collectionScope, entityId ) )
- .map(markStart).doOnNext(markCommit).map( new Func1<CollectionIoEvent<MvccEntity>,
- Void>() {
+ return Observable.from(new CollectionIoEvent<Id>(collectionScope, entityId)).map(markStart)
+ .doOnNext(markCommit).map(new Func1<CollectionIoEvent<MvccEntity>, Void>() {
@Override
- public Void call( final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent ) {
+ public Void call(final CollectionIoEvent<MvccEntity> mvccEntityCollectionIoEvent) {
return null;
}
- } );
+ });
}
@@@ -246,13 -319,30 +323,30 @@@
//wait for both to finish
Observable.merge(unique, optimistic).toBlocking().last();
+
-
}
});
}
-
-
+ @Override
-
+ public Observable<VersionSet> getLatestVersion( final Collection<Id> entityIds ) {
+
+ return Observable.create( new Observable.OnSubscribe<VersionSet>() {
+
+ @Override
+ public void call( final Subscriber<? super VersionSet> subscriber ) {
+ try {
+ final VersionSet logEntries = mvccLogEntrySerializationStrategy.load( collectionScope, entityIds,
+ UUIDGenerator.newTimeUUID() );
+
+ subscriber.onNext( logEntries );
+ subscriber.onCompleted();
+ }
+ catch ( Exception e ) {
+ subscriber.onError( e );
+ }
+ }
+ } );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
index 259e931,422a155..e518fb5
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTask.java
@@@ -23,14 -5,6 +22,14 @@@ import java.util.Iterator
import java.util.List;
import java.util.UUID;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
- import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
++import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValue;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
+import org.apache.usergrid.persistence.collection.serialization.impl.UniqueValueImpl;
+import org.apache.usergrid.persistence.model.entity.Entity;
+import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -38,9 -12,9 +37,7 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.event.EntityVersionDeleted;
import org.apache.usergrid.persistence.collection.mvcc.MvccEntitySerializationStrategy;
import org.apache.usergrid.persistence.collection.mvcc.MvccLogEntrySerializationStrategy;
- import org.apache.usergrid.persistence.collection.mvcc.entity.MvccLogEntry;
-import org.apache.usergrid.persistence.collection.MvccLogEntry;
import org.apache.usergrid.persistence.collection.serialization.SerializationFig;
--import org.apache.usergrid.persistence.collection.serialization.impl.LogEntryIterator;
import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.task.Task;
import org.apache.usergrid.persistence.model.entity.Id;
@@@ -124,78 -93,66 +121,78 @@@ public class EntityVersionCleanupTask i
@Override
public Void call() throws Exception {
-
-
- final UUID maxVersion = version;
-
-
- Observable<MvccLogEntry> versions = Observable.create( new ObservableIterator( "versionIterators" ) {
- @Override
- protected Iterator getIterator() {
- return new LogEntryIterator( logEntrySerializationStrategy, scope, entityId, maxVersion,
- serializationFig.getBufferSize() );
- }
- } );
-
-
- //get the uuid from the version
- versions.map( new Func1<MvccLogEntry, UUID>() {
- @Override
- public UUID call( final MvccLogEntry mvccLogEntry ) {
- return mvccLogEntry.getVersion();
- }
- } )
- //buffer our versions
- .buffer( serializationFig.getBufferSize() )
- //for each buffer set, delete all of them
- .doOnNext( new Action1<List<UUID>>() {
- @Override
- public void call( final List<UUID> versions ) {
-
- //Fire all the listeners
- fireEvents( versions );
-
- MutationBatch entityBatch = keyspace.prepareMutationBatch();
- MutationBatch logBatch = keyspace.prepareMutationBatch();
-
- for ( UUID version : versions ) {
- final MutationBatch entityDelete = entitySerializationStrategy.delete( scope, entityId, version );
-
- entityBatch.mergeShallow( entityDelete );
-
- final MutationBatch logDelete = logEntrySerializationStrategy.delete( scope, entityId, version );
-
- logBatch.mergeShallow( logDelete );
- }
-
-
- try {
- entityBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to delete entities in cleanup", e );
- }
-
- try {
- logBatch.execute();
- }
- catch ( ConnectionException e ) {
- throw new RuntimeException( "Unable to delete entities from the log", e );
- }
- }
- } ).count().toBlocking().last();
+ //TODO Refactor this logic into a a class that can be invoked from anywhere
+ //load every entity we have history of
+ Observable<List<MvccEntity>> deleteFieldsObservable =
+ Observable.create(new ObservableIterator<MvccEntity>("deleteColumns") {
+ @Override
+ protected Iterator<MvccEntity> getIterator() {
+ Iterator<MvccEntity> entities = entitySerializationStrategy.load(scope, entityId, version, serializationFig.getBufferSize());
+ return entities;
+ }
+ }) //buffer them for efficiency
+ .skip(1)
+ .buffer(serializationFig.getBufferSize()).doOnNext(
+ new Action1<List<MvccEntity>>() {
+ @Override
+ public void call(final List<MvccEntity> mvccEntities) {
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+ final MutationBatch entityBatch = keyspace.prepareMutationBatch();
+ final MutationBatch logBatch = keyspace.prepareMutationBatch();
+
+ for (MvccEntity mvccEntity : mvccEntities) {
+ if (!mvccEntity.getEntity().isPresent()) {
+ continue;
+ }
+
+ final UUID entityVersion = mvccEntity.getVersion();
+ final Entity entity = mvccEntity.getEntity().get();
+
+ //remove all unique fields from the index
+ for (final Field field : entity.getFields()) {
+ if (!field.isUnique()) {
+ continue;
+ }
- final UniqueValue unique = new UniqueValueImpl(scope, field, entityId, entityVersion);
- final MutationBatch deleteMutation = uniqueValueSerializationStrategy.delete(unique);
++ final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion);
++ final MutationBatch deleteMutation = uniqueValueSerializationStrategy.delete(scope,unique);
+ batch.mergeShallow(deleteMutation);
+ }
+
+ final MutationBatch entityDelete = entitySerializationStrategy.delete(scope, entityId, mvccEntity.getVersion());
+ entityBatch.mergeShallow(entityDelete);
+ final MutationBatch logDelete = logEntrySerializationStrategy.delete(scope, entityId, version);
+ logBatch.mergeShallow(logDelete);
+ }
+
+ try {
+ batch.execute();
+ } catch (ConnectionException e1) {
+ throw new RuntimeException("Unable to execute " +
+ "unique value " +
+ "delete", e1);
+ }
+ fireEvents(mvccEntities);
+ try {
+ entityBatch.execute();
+ } catch (ConnectionException e) {
+ throw new RuntimeException("Unable to delete entities in cleanup", e);
+ }
+
+ try {
+ logBatch.execute();
+ } catch (ConnectionException e) {
+ throw new RuntimeException("Unable to delete entities from the log", e);
+ }
+
+ }
+ }
+
+
+ );
+
+ final int removedCount = deleteFieldsObservable.count().toBlocking().last();
+
+ LOG.debug("Removed unique values for {} entities of entity {}",removedCount,entityId);
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
index b54f375,9e2d52c..994465d
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/mvcc/stage/delete/MarkCommit.java
@@@ -119,5 -119,69 +119,72 @@@ public class MarkCommit implements Acti
catch ( ConnectionException e ) {
throw new RuntimeException( "Unable to mark entry as deleted" );
}
-
-
- //TODO Refactor this logic into a a class that can be invoked from anywhere
- //load every entity we have history of
- Observable<List<MvccEntity>> deleteFieldsObservable =
- Observable.create( new ObservableIterator<MvccEntity>( "deleteColumns" ) {
- @Override
- protected Iterator<MvccEntity> getIterator() {
- Iterator<MvccEntity> entities =
- entityStrat.load( collectionScope, entityId, entity.getVersion(), 100 );
-
- return entities;
- }
- } ) //buffer them for efficiency
- .buffer( serializationFig.getBufferSize() ).doOnNext(
-
- new Action1<List<MvccEntity>>() {
- @Override
- public void call( final List<MvccEntity> mvccEntities ) {
-
-
- final MutationBatch batch = keyspace.prepareMutationBatch();
-
- for ( MvccEntity mvccEntity : mvccEntities ) {
- if ( !mvccEntity.getEntity().isPresent() ) {
- continue;
- }
-
- final UUID entityVersion = mvccEntity.getVersion();
-
- final Entity entity = mvccEntity.getEntity().get();
-
- //remove all unique fields from the index
- for ( final Field field : entity.getFields() ) {
-
- if(!field.isUnique()){
- continue;
- }
-
- final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
-
- final MutationBatch deleteMutation = uniqueValueStrat.delete(collectionScope, unique );
-
- batch.mergeShallow( deleteMutation );
- }
- }
-
- try {
- batch.execute();
- }
- catch ( ConnectionException e1 ) {
- throw new RuntimeException( "Unable to execute " +
- "unique value " +
- "delete", e1 );
- }
- }
- }
-
-
- );
-
- final int removedCount = deleteFieldsObservable.count().toBlocking().last();
-
- LOG.debug("Removed unique values for {} entities of entity {}", removedCount, entityId );
++//<<<<<<< HEAD
++//=======
++//
++//
++// //TODO Refactor this logic into a a class that can be invoked from anywhere
++// //load every entity we have history of
++// Observable<List<MvccEntity>> deleteFieldsObservable =
++// Observable.create( new ObservableIterator<MvccEntity>( "deleteColumns" ) {
++// @Override
++// protected Iterator<MvccEntity> getIterator() {
++// Iterator<MvccEntity> entities =
++// entityStrat.load( collectionScope, entityId, entity.getVersion(), 100 );
++//
++// return entities;
++// }
++// } ) //buffer them for efficiency
++// .buffer( serializationFig.getBufferSize() ).doOnNext(
++//
++// new Action1<List<MvccEntity>>() {
++// @Override
++// public void call( final List<MvccEntity> mvccEntities ) {
++//
++//
++// final MutationBatch batch = keyspace.prepareMutationBatch();
++//
++// for ( MvccEntity mvccEntity : mvccEntities ) {
++// if ( !mvccEntity.getEntity().isPresent() ) {
++// continue;
++// }
++//
++// final UUID entityVersion = mvccEntity.getVersion();
++//
++// final Entity entity = mvccEntity.getEntity().get();
++//
++// //remove all unique fields from the index
++// for ( final Field field : entity.getFields() ) {
++//
++// if(!field.isUnique()){
++// continue;
++// }
++//
++// final UniqueValue unique = new UniqueValueImpl( field, entityId, entityVersion );
++//
++// final MutationBatch deleteMutation = uniqueValueStrat.delete(collectionScope, unique );
++//
++// batch.mergeShallow( deleteMutation );
++// }
++// }
++//
++// try {
++// batch.execute();
++// }
++// catch ( ConnectionException e1 ) {
++// throw new RuntimeException( "Unable to execute " +
++// "unique value " +
++// "delete", e1 );
++// }
++// }
++// }
++//
++//
++// );
++//
++// final int removedCount = deleteFieldsObservable.count().toBlocking().last();
++//
++// LOG.debug("Removed unique values for {} entities of entity {}", removedCount, entityId );
++//>>>>>>> befcdcab6f7f1c83dbcb2a24eddb055c9297d59f
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersionSerializer.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersionSerializer.java
index 810a1fc,dc4efde..ce6512f
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersionSerializer.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/EntityVersionSerializer.java
@@@ -24,6 -24,10 +24,7 @@@ import java.util.UUID
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.usergrid.persistence.collection.mvcc.entity.MvccValidationUtils;
-import org.apache.usergrid.persistence.collection.util.EntityUtils;
-import org.apache.usergrid.persistence.core.util.ValidationUtils;
+ import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d9ace9e0/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
index 4c1c75f,1fce6e2..02233a6
--- a/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
+++ b/stack/corepersistence/collection/src/test/java/org/apache/usergrid/persistence/collection/impl/EntityVersionCleanupTaskTest.java
@@@ -27,8 -27,6 +27,8 @@@ import java.util.concurrent.CountDownLa
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Semaphore;
- import org.apache.usergrid.persistence.collection.mvcc.entity.MvccEntity;
++import org.apache.usergrid.persistence.collection.MvccEntity;
+import org.apache.usergrid.persistence.collection.serialization.UniqueValueSerializationStrategy;
import org.junit.AfterClass;
import org.junit.Test;