You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2015/11/20 05:26:10 UTC
[01/19] usergrid git commit: Add back in shard meta removal.
Repository: usergrid
Updated Branches:
refs/heads/master 6f3541be6 -> 6fffe936b
Add back in shard meta removal.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/afffdf27
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/afffdf27
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/afffdf27
Branch: refs/heads/master
Commit: afffdf278242ccb6e309cff220af371f32a0c0e4
Parents: 8ed1bc7
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Nov 16 08:34:01 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Nov 16 08:34:01 2015 -0800
----------------------------------------------------------------------
.../impl/shard/impl/ShardGroupDeletionImpl.java | 168 +++++++++----------
1 file changed, 84 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/afffdf27/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index 4d6be1b..d289f30 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -125,90 +125,90 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
final ShardEntryGroup shardEntryGroup,
final Iterator<MarkedEdge> edgeIterator ) {
- //TEMPORARILY Removed until further testing
- return DeleteResult.NO_OP;
-
-// logger.trace( "Beginning audit of shard group {}", shardEntryGroup );
-//
-// /**
-// * Compaction is pending, we cannot check it
-// */
-// if ( shardEntryGroup.isCompactionPending() ) {
-// logger.trace( "Shard group {} is compacting, not auditing group", shardEntryGroup );
-// return DeleteResult.COMPACTION_PENDING;
-// }
-//
-//
-// final long currentTime = timeService.getCurrentTime();
-//
-// if ( shardEntryGroup.isNew( currentTime ) ) {
-// logger.trace( "Shard group {} contains a shard that is is too new, not auditing group", shardEntryGroup );
-// return DeleteResult.TOO_NEW;
-// }
-//
-// /**
-// * We have edges, and therefore cannot delete them
-// */
-// if ( edgeIterator.hasNext() ) {
-// logger.trace( "Shard group {} has edges, not deleting", shardEntryGroup );
-//
-// return DeleteResult.CONTAINS_EDGES;
-// }
-//
-//
-// //now we can proceed based on the shard meta state and we don't have any edge
-//
-// DeleteResult result = DeleteResult.NO_OP;
-//
-// MutationBatch rollup = null;
-//
-// for ( final Shard shard : shardEntryGroup.getReadShards() ) {
-//
-// //skip the min shard
-// if(shard.isMinShard()){
-// logger.trace( "Shard {} in group {} is the minimum, not deleting", shard, shardEntryGroup );
-// continue;
-// }
-//
-// //The shard is not compacted, we cannot remove it. This should never happen, a bit of an "oh shit" scenario.
-// //the isCompactionPending should return false in this case
-// if(!shard.isCompacted()){
-// logger.warn( "Shard {} in group {} is not compacted yet was checked. Short circuiting", shard, shardEntryGroup );
-// return DeleteResult.NO_OP;
-// }
-//
-//
-// final MutationBatch shardRemovalMutation =
-// edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
-//
-// if ( rollup == null ) {
-// rollup = shardRemovalMutation;
-// }
-//
-// else {
-// rollup.mergeShallow( shardRemovalMutation );
-// }
-//
-// result = DeleteResult.DELETED;
-//
-// logger.trace( "Removing shard {} in group {}", shard, shardEntryGroup );
-// }
-//
-//
-// if( rollup != null) {
-//
-// try {
-// rollup.execute();
-// }
-// catch ( ConnectionException e ) {
-// logger.error( "Unable to execute shard deletion", e );
-// throw new RuntimeException( "Unable to execute shard deletion", e );
-// }
-// }
-//
-// logger.trace( "Completed auditing shard group {}", shardEntryGroup );
-//
-// return result;
+ //Use ths to TEMPORARILY remove deletes from occurring
+ //return DeleteResult.NO_OP;
+
+ logger.trace( "Beginning audit of shard group {}", shardEntryGroup );
+
+ /**
+ * Compaction is pending, we cannot check it
+ */
+ if ( shardEntryGroup.isCompactionPending() ) {
+ logger.trace( "Shard group {} is compacting, not auditing group", shardEntryGroup );
+ return DeleteResult.COMPACTION_PENDING;
+ }
+
+
+ final long currentTime = timeService.getCurrentTime();
+
+ if ( shardEntryGroup.isNew( currentTime ) ) {
+ logger.trace( "Shard group {} contains a shard that is is too new, not auditing group", shardEntryGroup );
+ return DeleteResult.TOO_NEW;
+ }
+
+ /**
+ * We have edges, and therefore cannot delete them
+ */
+ if ( edgeIterator.hasNext() ) {
+ logger.trace( "Shard group {} has edges, not deleting", shardEntryGroup );
+
+ return DeleteResult.CONTAINS_EDGES;
+ }
+
+
+ //now we can proceed based on the shard meta state and we don't have any edge
+
+ DeleteResult result = DeleteResult.NO_OP;
+
+ MutationBatch rollup = null;
+
+ for ( final Shard shard : shardEntryGroup.getReadShards() ) {
+
+ //skip the min shard
+ if(shard.isMinShard()){
+ logger.trace( "Shard {} in group {} is the minimum, not deleting", shard, shardEntryGroup );
+ continue;
+ }
+
+ //The shard is not compacted, we cannot remove it. This should never happen, a bit of an "oh shit" scenario.
+ //the isCompactionPending should return false in this case
+ if(!shard.isCompacted()){
+ logger.warn( "Shard {} in group {} is not compacted yet was checked. Short circuiting", shard, shardEntryGroup );
+ return DeleteResult.NO_OP;
+ }
+
+
+ final MutationBatch shardRemovalMutation =
+ edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
+
+ if ( rollup == null ) {
+ rollup = shardRemovalMutation;
+ }
+
+ else {
+ rollup.mergeShallow( shardRemovalMutation );
+ }
+
+ result = DeleteResult.DELETED;
+
+ logger.info( "Removing shard {} in group {}", shard, shardEntryGroup );
+ }
+
+
+ if( rollup != null) {
+
+ try {
+ rollup.execute();
+ }
+ catch ( ConnectionException e ) {
+ logger.error( "Unable to execute shard deletion", e );
+ throw new RuntimeException( "Unable to execute shard deletion", e );
+ }
+ }
+
+ logger.trace( "Completed auditing shard group {}", shardEntryGroup );
+
+ return result;
}
[12/19] usergrid git commit: Fix delete so that it doesn't return null
Posted by mr...@apache.org.
Fix delete so that it doesn't return null
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/88f6feae
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/88f6feae
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/88f6feae
Branch: refs/heads/master
Commit: 88f6feae8c8d635598954ee9a5d239eda42653bd
Parents: 9bc2241
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 09:43:36 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 09:43:36 2015 -0800
----------------------------------------------------------------------
.../asyncevents/EventBuilderImpl.java | 39 +++++++++-----------
1 file changed, 17 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/88f6feae/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index d819f39..9b68c4c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -125,42 +125,37 @@ public class EventBuilderImpl implements EventBuilder {
log.debug( "Deleting entity id from index in app scope {} with entityId {} }", applicationScope, entityId );
final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope );
-
final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ //TODO USERGRID-1123: Implement so we don't iterate logs twice (latest DELETED version, then to get all DELETED)
- //TODO: change this to be an observable
- //so we get these versions and loop through them until we find the MvccLogEntry that is marked as delete.
- //TODO: evauluate this to possibly be an observable to pass to the nextmethod.
MvccLogEntry mostRecentlyMarked = ecm.getVersions( entityId ).toBlocking()
.firstOrDefault( null,
mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
- //If there is nothing marked then we shouldn't return any results.
- //TODO: evaluate if we want to return null or return empty observable when we don't have any results marked as deleted.
- if(mostRecentlyMarked == null)
- return null;
+ // De-indexing and entity deletes don't check log entiries. We must do that first. If no DELETED logs, then
+ // return an empty observable as our no-op.
+ Observable<IndexOperationMessage> deIndexObservable = Observable.empty();
+ Observable<List<MvccLogEntry>> ecmDeleteObservable = Observable.empty();
- //observable of index operation messages
- //this method will need the most recent version.
- //When we go to compact the graph make sure you turn on the debugging mode for the deleted nodes so
- //we can verify that we mark them. That said that part seems kinda done. as we also delete the mvcc buffers.
- final Observable<IndexOperationMessage> edgeObservable =
- indexService.deleteEntityIndexes( applicationScope, entityId, mostRecentlyMarked.getVersion() );
+ if(mostRecentlyMarked != null){
+ deIndexObservable =
+ indexService.deleteEntityIndexes( applicationScope, entityId, mostRecentlyMarked.getVersion() );
+ ecmDeleteObservable =
+ ecm.getVersions( entityId )
+ .filter( mvccLogEntry-> mvccLogEntry.getState() == MvccLogEntry.State.DELETED)
+ .buffer( serializationFig.getBufferSize() )
+ .doOnNext( buffer -> ecm.delete( buffer ) );
- //TODO: not sure what we need the list of versions here when we search for the mark above
- //observable of entries as the batches are deleted
- final Observable<List<MvccLogEntry>> entries =
- ecm.getVersions( entityId ).buffer( serializationFig.getBufferSize() )
- .doOnNext( buffer -> ecm.delete( buffer ) );
+ }
- // observable of the edge delete from graph
- final Observable<Id> compactedNode = gm.compactNode(entityId);
+ // Graph compaction checks the versions inside compactNode, just build this up for the caller to subscribe to
+ final Observable<Id> graphCompactObservable = gm.compactNode(entityId);
- return new EntityDeleteResults( edgeObservable, entries, compactedNode );
+ return new EntityDeleteResults( deIndexObservable, ecmDeleteObservable, graphCompactObservable );
}
[17/19] usergrid git commit: format change only
Posted by mr...@apache.org.
format change only
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e5b97304
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e5b97304
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e5b97304
Branch: refs/heads/master
Commit: e5b97304ebe2dcccce32ea18c7d7994130ca5738
Parents: be9bcb0
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Nov 19 15:29:39 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Nov 19 15:29:39 2015 -0500
----------------------------------------------------------------------
.../corepersistence/asyncevents/EventBuilderImpl.java | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e5b97304/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 48df8d0..d624132 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -131,8 +131,7 @@ public class EventBuilderImpl implements EventBuilder {
//TODO USERGRID-1123: Implement so we don't iterate logs twice (latest DELETED version, then to get all DELETED)
MvccLogEntry mostRecentlyMarked = ecm.getVersions( entityId ).toBlocking()
- .firstOrDefault( null,
- mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
+ .firstOrDefault( null, mvccLogEntry -> mvccLogEntry.getState() == MvccLogEntry.State.DELETED );
// De-indexing and entity deletes don't check log entiries. We must do that first. If no DELETED logs, then
// return an empty observable as our no-op.
@@ -145,14 +144,12 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) <= 0)
+ .filter( mvccLogEntry->
+ UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) <= 0)
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
-
-
}
-
// Graph compaction checks the versions inside compactNode, just build this up for the caller to subscribe to
final Observable<Id> graphCompactObservable = gm.compactNode(entityId);
[14/19] usergrid git commit: Don't filter out properties that have
name of 'size'. The internal size prop is noted in our metadata object,
not at the root fields level.
Posted by mr...@apache.org.
Don't filter out properties that have name of 'size'. The internal size prop is noted in our metadata object, not at the root fields level.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/aea92641
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/aea92641
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/aea92641
Branch: refs/heads/master
Commit: aea926411c29e57e6d65cd50cbacf921821b16e1
Parents: faa234a
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 11:07:55 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 11:07:55 2015 -0800
----------------------------------------------------------------------
.../usergrid/persistence/model/entity/MapToEntityConverter.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/aea92641/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
index 2357a57..a225820 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
@@ -50,9 +50,7 @@ public class MapToEntityConverter{
public Entity fromMap(final Entity entity,final Map<String, Object> map,final SchemaManager schemaManager, final String entityType, boolean topLevel) {
for ( String fieldName : map.keySet() ) {
- if(fieldName.equals("size")){
- continue;
- }
+
Object value = map.get( fieldName );
boolean unique = schemaManager == null ? topLevel : topLevel && schemaManager.isPropertyUnique(entityType, fieldName);
//cannot store fields that aren't locations
[02/19] usergrid git commit: Added test to prove shard ordering.
Posted by mr...@apache.org.
Added test to prove shard ordering.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2b2793ff
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2b2793ff
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2b2793ff
Branch: refs/heads/master
Commit: 2b2793ffeddb0836ced383b13c25d8f86432b632
Parents: afffdf2
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 16 19:00:20 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 16 19:00:20 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/ShardTest.java | 65 ++++++++++++++++++++
1 file changed, 65 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2b2793ff/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java
new file mode 100644
index 0000000..478873f
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardTest.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard;
+
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertTrue;
+
+
+public class ShardTest {
+
+ @Test
+ public void testShardIndexOrder() {
+ final Shard first = new Shard( 1, 1, false );
+
+
+ final Shard second = new Shard( 2, 1, false );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+
+
+ @Test
+ public void testShardTimestapOrder() {
+ final Shard first = new Shard( 1, 1, false );
+
+
+ final Shard second = new Shard( 1, 2, false );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+
+
+ @Test
+ public void testShardCompactedOrder() {
+ final Shard first = new Shard( 1, 1, false );
+
+
+ final Shard second = new Shard( 1, 1, true );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+}
[11/19] usergrid git commit: Fixes issue that caused shards to get
removed prematurely.
Posted by mr...@apache.org.
Fixes issue that caused shards to get removed prematurely.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9bc22410
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9bc22410
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9bc22410
Branch: refs/heads/master
Commit: 9bc22410dd785482f715faec4cbfeae0e712a502
Parents: 784fe51
Author: Michael Russo <mi...@gmail.com>
Authored: Wed Nov 18 15:48:07 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Wed Nov 18 15:48:07 2015 -0800
----------------------------------------------------------------------
.../impl/EdgeSerializationImpl.java | 61 ++++++++++++++++++--
.../shard/impl/EdgeShardSerializationImpl.java | 3 +-
.../shard/impl/ShardGroupColumnIterator.java | 4 +-
.../shard/impl/ShardGroupCompactionImpl.java | 2 +-
.../impl/shard/EdgeShardSerializationTest.java | 44 ++++++++++++++
5 files changed, 106 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
index 9e25946..0f4d722 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeSerializationImpl.java
@@ -26,15 +26,15 @@ import java.util.UUID;
import javax.inject.Inject;
+import com.google.common.base.Optional;
import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.consistency.TimeService;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
-import org.apache.usergrid.persistence.graph.GraphFig;
-import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
-import org.apache.usergrid.persistence.graph.SearchByIdType;
+import org.apache.usergrid.persistence.graph.*;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
+import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
@@ -303,6 +303,15 @@ public class EdgeSerializationImpl implements EdgeSerialization {
protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) {
return shardedEdgeSerialization.getEdgeVersions( edgeColumnFamilies, scope, search, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByEdge searchFullRange = new SimpleSearchByEdge(
+ search.sourceNode(), search.getType(),search.targetNode(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent());
+
+ return shardedEdgeSerialization.getEdgeVersions( edgeColumnFamilies, scope, searchFullRange, readShards );
+ }
};
}
@@ -329,6 +338,15 @@ public class EdgeSerializationImpl implements EdgeSerialization {
protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) {
return shardedEdgeSerialization.getEdgesFromSource( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByEdgeType searchFullRange = new SimpleSearchByEdgeType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, Optional.absent(), false );
+
+ return shardedEdgeSerialization.getEdgesFromSource( edgeColumnFamilies, scope, searchFullRange, readShards );
+ }
};
}
@@ -357,6 +375,17 @@ public class EdgeSerializationImpl implements EdgeSerialization {
return shardedEdgeSerialization
.getEdgesFromSourceByTargetType( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange (final Collection<Shard> readShards) {
+
+ final SearchByIdType edgeTypeFullRange = new SimpleSearchByIdType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ edgeType.getIdType(), Optional.absent(), false );
+
+ return shardedEdgeSerialization
+ .getEdgesFromSourceByTargetType( edgeColumnFamilies, scope, edgeTypeFullRange, readShards);
+ }
};
}
@@ -382,6 +411,17 @@ public class EdgeSerializationImpl implements EdgeSerialization {
protected Iterator<MarkedEdge> getIterator( final Collection<Shard> readShards ) {
return shardedEdgeSerialization.getEdgesToTarget( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByEdgeType edgeTypeFullRange = new SimpleSearchByEdgeType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ Optional.absent(), false );
+
+ return shardedEdgeSerialization.getEdgesToTarget( edgeColumnFamilies, scope, edgeTypeFullRange, readShards );
+ }
+
};
}
@@ -411,6 +451,17 @@ public class EdgeSerializationImpl implements EdgeSerialization {
return shardedEdgeSerialization
.getEdgesToTargetBySourceType( edgeColumnFamilies, scope, edgeType, readShards );
}
+
+ @Override
+ protected Iterator<MarkedEdge> getIteratorFullRange( final Collection<Shard> readShards ) {
+
+ final SearchByIdType edgeTypeFullRange = new SimpleSearchByIdType(
+ edgeType.getNode(), edgeType.getType(), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING,
+ edgeType.getIdType(), Optional.absent(), false );
+
+ return shardedEdgeSerialization.getEdgesToTargetBySourceType( edgeColumnFamilies, scope, edgeTypeFullRange, readShards);
+
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
index f107307..2f83d6f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
@@ -156,7 +156,8 @@ public class EdgeShardSerializationImpl implements EdgeShardSerialization {
final MutationBatch batch = keyspace.prepareMutationBatch();
- batch.withRow( EDGE_SHARDS, rowKey ).deleteColumn( shard.getShardIndex() );
+ batch.withTimestamp(shard.getCreatedTime()).withRow( EDGE_SHARDS, rowKey )
+ .deleteColumn( shard.getShardIndex() );
return batch;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
index 72b617f..9604e63 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupColumnIterator.java
@@ -117,6 +117,8 @@ public abstract class ShardGroupColumnIterator implements Iterator<MarkedEdge> {
*/
protected abstract Iterator<MarkedEdge> getIterator( Collection<Shard> readShards );
+ protected abstract Iterator<MarkedEdge> getIteratorFullRange( Collection<Shard> readShards );
+
public boolean advance() {
@@ -141,7 +143,7 @@ public abstract class ShardGroupColumnIterator implements Iterator<MarkedEdge> {
logger.trace( "Our shard is empty, we need to perform an audit on shard group {}", group );
//fire and forget if we miss it here, we'll get it next read
- shardGroupDeletion.maybeDeleteShard(this.applicationScope, this.directedEdgeMeta, group, getIterator( group.getReadShards() ) );
+ shardGroupDeletion.maybeDeleteShard(this.applicationScope, this.directedEdgeMeta, group, getIteratorFullRange( group.getReadShards() ) );
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 7dd0521..e663d5a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -150,7 +150,7 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
.checkArgument( group.shouldCompact( startTime ), "Compaction cannot be run yet. Ignoring compaction." );
if(LOG.isDebugEnabled()) {
- LOG.debug("Compacting shard group. count is {} ", countAudits.get());
+ LOG.debug("Compacting shard group. Audit count is {} ", countAudits.get());
}
final CompactionResult.CompactionBuilder resultBuilder = CompactionResult.builder();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bc22410/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
index 2641ed7..1f8bfa9 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
@@ -40,6 +40,8 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import static org.apache.usergrid.persistence.core.util.IdGenerator.createId;
import static org.junit.Assert.assertEquals;
@@ -52,6 +54,8 @@ import static org.mockito.Mockito.when;
@UseModules({ TestGraphModule.class })
public class EdgeShardSerializationTest {
+ private static final Logger log = LoggerFactory.getLogger( EdgeShardSerializationTest.class );
+
@Inject
@Rule
@@ -203,4 +207,44 @@ public class EdgeShardSerializationTest {
assertFalse( results.hasNext() );
}
+
+ @Test
+ public void sameShardIndexRemoval() throws ConnectionException {
+
+ final Id now = IdGenerator.createId( "test" );
+
+ final long timestamp = 2000L;
+
+ final Shard shard1 = new Shard( 1000L, timestamp, false );
+ final Shard shard2 = new Shard( shard1.getShardIndex(), timestamp * 2, true );
+
+
+ final DirectedEdgeMeta sourceEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType( now, "edgeType", "subType" );
+ MutationBatch batch = edgeShardSerialization.writeShardMeta( scope, shard1, sourceEdgeMeta );
+ batch.mergeShallow( edgeShardSerialization.writeShardMeta( scope, shard2, sourceEdgeMeta ) );
+ batch.execute();
+
+
+ Iterator<Shard> results =
+ edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
+
+ // Latest timestamp comes first
+ assertEquals( shard2, results.next() );
+
+ // This should now not remove anything
+ edgeShardSerialization.removeShardMeta( scope, shard1, sourceEdgeMeta ).execute();
+
+
+ // Get iterator again
+ results = edgeShardSerialization.getShardMetaData( scope, Optional.<Shard>absent(), sourceEdgeMeta );
+
+ // We should still have shard3 stored
+ assertEquals( shard2, results.next() );
+
+
+
+ }
+
+
+
}
[03/19] usergrid git commit: Added additional test for isNew in
shardEntryGroup
Posted by mr...@apache.org.
Added additional test for isNew in shardEntryGroup
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3a6fd944
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3a6fd944
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3a6fd944
Branch: refs/heads/master
Commit: 3a6fd9444f7687f495f2d13fea2b6bb8ccc5d0e5
Parents: 2b2793f
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 16 19:08:04 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 16 19:08:04 2015 -0700
----------------------------------------------------------------------
.../impl/shard/ShardEntryGroupTest.java | 31 ++++++++++++++++++++
1 file changed, 31 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3a6fd944/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
index 28d42d2..2f0dfaa 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
@@ -434,6 +434,37 @@ public class ShardEntryGroupTest {
}
+
+
+ @Test
+ public void testIsNew() {
+ //set with no shard
+
+ final long delta = 10000;
+
+ //created at 10000
+ final Shard firstShard = new Shard( 10000, 10000, true );
+
+ final Shard secondShard = new Shard( 10000, 10001, true );
+
+ final ShardEntryGroup shardGroup = new ShardEntryGroup( delta );
+
+ shardGroup.addShard( secondShard );
+ shardGroup.addShard( firstShard );
+
+ final boolean resultCreateTime = shardGroup.isNew( secondShard.getCreatedTime() );
+
+ assertTrue( "This is a new shard", resultCreateTime );
+
+ final boolean resultEqualToDelta = shardGroup.isNew( secondShard.getCreatedTime() + delta );
+
+ assertTrue( "This is a new shard", resultEqualToDelta );
+
+
+ final boolean greaterThan = shardGroup.isNew( secondShard.getCreatedTime() + delta + 1 );
+
+ assertFalse( "This is not a new shard", greaterThan );
+ }
}
[05/19] usergrid git commit: Added logging to deletion task.
Posted by mr...@apache.org.
Added logging to deletion task.
Updated consistency test to have 2 read workers.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c44dfdab
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c44dfdab
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c44dfdab
Branch: refs/heads/master
Commit: c44dfdabfc23db4e0d2eed739d19f865c58288cd
Parents: ad12f35
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 09:07:34 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 09:07:34 2015 -0700
----------------------------------------------------------------------
.../impl/shard/impl/ShardGroupDeletionImpl.java | 9 +++--
.../graph/GraphManagerShardConsistencyIT.java | 40 +++++++++++---------
2 files changed, 28 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c44dfdab/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index c460a5d..373d42b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -138,6 +138,7 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
return DeleteResult.COMPACTION_PENDING;
}
+ logger.trace( "Shard group {} has no compaction pending", shardEntryGroup );
final long currentTime = timeService.getCurrentTime();
@@ -146,6 +147,8 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
return DeleteResult.TOO_NEW;
}
+ logger.trace( "Shard group {} has passed the delta timeout at {}", shardEntryGroup, currentTime );
+
/**
* We have edges, and therefore cannot delete them
*/
@@ -156,6 +159,9 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
+ logger.trace( "Shard group {} has no edges continuing to delete", shardEntryGroup, currentTime );
+
+
//now we can proceed based on the shard meta state and we don't have any edge
DeleteResult result = DeleteResult.NO_OP;
@@ -178,9 +184,6 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
- logger.info( "Deleting shard {} at time {} ", shard, timeService.getCurrentTime() );
-
-
final MutationBatch shardRemovalMutation =
edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c44dfdab/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index a9f9cb7..b903c6c 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -266,31 +266,35 @@ public class GraphManagerShardConsistencyIT {
final Meter readMeter = registry.meter( "readThroughput" );
- /**
- * Start reading continuously while we migrate data to ensure our view is always correct
- */
- final ListenableFuture<Long> future =
- executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
-
final List<Throwable> failures = new ArrayList<>();
+ for(int i = 0; i < 2; i ++) {
- //add the future
- Futures.addCallback( future, new FutureCallback<Long>() {
- @Override
- public void onSuccess( @Nullable final Long result ) {
- log.info( "Successfully ran the read, re-running" );
+ /**
+ * Start reading continuously while we migrate data to ensure our view is always correct
+ */
+ final ListenableFuture<Long> future =
executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
- }
- @Override
- public void onFailure( final Throwable t ) {
- failures.add( t );
- log.error( "Failed test!", t );
- }
- } );
+ //add the future
+ Futures.addCallback( future, new FutureCallback<Long>() {
+
+ @Override
+ public void onSuccess( @Nullable final Long result ) {
+ log.info( "Successfully ran the read, re-running" );
+ executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
+ }
+
+
+ @Override
+ public void onFailure( final Throwable t ) {
+ failures.add( t );
+ log.error( "Failed test!", t );
+ }
+ } );
+ }
int compactedCount;
[07/19] usergrid git commit: Fixes log statement
Posted by mr...@apache.org.
Fixes log statement
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/06c9c101
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/06c9c101
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/06c9c101
Branch: refs/heads/master
Commit: 06c9c101242c125e4832f89bf6e6884174cb782e
Parents: 4f00a43
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 10:54:52 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 10:54:52 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/impl/ShardGroupCompactionImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/06c9c101/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index 21f2d72..7dd0521 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -307,8 +307,9 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
countAudits.getAndIncrement();
if(LOG.isDebugEnabled()) {
- LOG.debug("Auditing shard group. count is {} ", countAudits.get());
+ LOG.debug("Auditing shard group {}. count is {} ", group, countAudits.get());
}
+
/**
* Try and submit. During back pressure, we may not be able to submit, that's ok. Better to drop than to
* hose the system
[19/19] usergrid git commit: Merge branch 'release' of
https://git-wip-us.apache.org/repos/asf/usergrid
Posted by mr...@apache.org.
Merge branch 'release' of https://git-wip-us.apache.org/repos/asf/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6fffe936
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6fffe936
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6fffe936
Branch: refs/heads/master
Commit: 6fffe936bbdfa188892e8e8a9190eb03ec9c17a5
Parents: 6f3541b 9bfbc53
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 20:25:43 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 20:25:43 2015 -0800
----------------------------------------------------------------------
.../asyncevents/EventBuilderImpl.java | 47 +++--
.../usergrid/persistence/graph/GraphFig.java | 2 +-
.../impl/EdgeSerializationImpl.java | 61 ++++++-
.../shard/impl/EdgeShardSerializationImpl.java | 3 +-
.../shard/impl/NodeShardAllocationImpl.java | 11 +-
.../shard/impl/ShardGroupColumnIterator.java | 4 +-
.../shard/impl/ShardGroupCompactionImpl.java | 5 +-
.../impl/shard/impl/ShardGroupDeletionImpl.java | 174 ++++++++++---------
.../graph/GraphManagerShardConsistencyIT.java | 40 +++--
.../impl/shard/EdgeShardSerializationTest.java | 44 +++++
.../impl/shard/NodeShardAllocationTest.java | 6 +-
.../impl/shard/ShardEntryGroupTest.java | 31 ++++
.../serialization/impl/shard/ShardTest.java | 65 +++++++
.../model/entity/MapToEntityConverter.java | 7 +-
14 files changed, 351 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6fffe936/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
[06/19] usergrid git commit: Fixes logging statement
Posted by mr...@apache.org.
Fixes logging statement
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4f00a437
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4f00a437
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4f00a437
Branch: refs/heads/master
Commit: 4f00a437e556d8c352107a58767fb19590b62875
Parents: c44dfda
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 10:06:54 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 10:06:54 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/impl/NodeShardAllocationImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4f00a437/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index 374d373..0cc12b6 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -226,7 +226,7 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
* Sanity check in case we audit before we have a full shard
*/
if ( marked == null ) {
- LOG.trace( "Shard {} in shard group {} not full, not splitting", shardEntryGroup );
+ LOG.trace( "Shard {} in shard group {} not full, not splitting", shard, shardEntryGroup );
return false;
}
[04/19] usergrid git commit: Added additional logging statement
Posted by mr...@apache.org.
Added additional logging statement
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ad12f359
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ad12f359
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ad12f359
Branch: refs/heads/master
Commit: ad12f3599d2d66e799f6c3294f6e33d8e1c5b4d4
Parents: 3a6fd94
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 08:30:50 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 08:30:50 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/impl/ShardGroupDeletionImpl.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ad12f359/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
index d289f30..c460a5d 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupDeletionImpl.java
@@ -178,6 +178,9 @@ public class ShardGroupDeletionImpl implements ShardGroupDeletion {
}
+ logger.info( "Deleting shard {} at time {} ", shard, timeService.getCurrentTime() );
+
+
final MutationBatch shardRemovalMutation =
edgeShardSerialization.removeShardMeta( applicationScope, shard, directedEdgeMeta );
[08/19] usergrid git commit: Adds more logging
Posted by mr...@apache.org.
Adds more logging
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a33f616b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a33f616b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a33f616b
Branch: refs/heads/master
Commit: a33f616beed86f475999a5904520776494891968
Parents: 06c9c10
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 11:17:07 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 11:17:07 2015 -0700
----------------------------------------------------------------------
.../impl/shard/impl/NodeShardAllocationImpl.java | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a33f616b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index 0cc12b6..d52f807 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -138,11 +138,13 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
* Nothing to do, it's been created very recently, we don't create a new one
*/
if ( shardEntryGroup.isCompactionPending() ) {
+ LOG.trace( "Shard entry group {} is compacting, not auditing", shardEntryGroup );
return false;
}
//we can't allocate, we have more than 1 write shard currently. We need to compact first
if ( shardEntryGroup.entrySize() != 1 ) {
+ LOG.trace( "Shard entry group {} does not have 1 entry, not allocating", shardEntryGroup );
return false;
}
@@ -151,9 +153,12 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
* Check the min shard in our system
*/
final Shard shard = shardEntryGroup.getMinShard();
+ final long minTime = getMinTime();
- if ( shard.getCreatedTime() >= getMinTime() ) {
+
+ if ( shard.getCreatedTime() >= minTime ) {
+ LOG.trace( "Shard entry group {} and shard {} is before the minimum created time of {}. Not allocating.does not have 1 entry, not allocating", shardEntryGroup, shard, minTime );
return false;
}
@@ -253,7 +258,7 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
@Override
public long getMinTime() {
- final long minimumAllowed = 2 * graphFig.getShardCacheTimeout();
+ final long minimumAllowed = ( long ) (2.5 * graphFig.getShardCacheTimeout());
final long minDelta = graphFig.getShardMinDelta();
[13/19] usergrid git commit: Update filter to allow removal of all
entities older than the last marked version.
Posted by mr...@apache.org.
Update filter to allow removal of all entities older than the last marked version.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/faa234a8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/faa234a8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/faa234a8
Branch: refs/heads/master
Commit: faa234a85f1526a17a6f7d3049a7cb0820816190
Parents: 88f6fea
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 10:51:22 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 10:51:22 2015 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/asyncevents/EventBuilderImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/faa234a8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 9b68c4c..f7f7ba3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -144,7 +144,7 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> mvccLogEntry.getState() == MvccLogEntry.State.DELETED)
+ .filter( mvccLogEntry-> mvccLogEntry.getVersion().timestamp() <= mostRecentlyMarked.getVersion().timestamp())
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
[10/19] usergrid git commit: Fixes incorrect default property
Posted by mr...@apache.org.
Fixes incorrect default property
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/784fe51c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/784fe51c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/784fe51c
Branch: refs/heads/master
Commit: 784fe51ca9593f813294865e198d1fd07b796aee
Parents: 8ba0ef0
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 11:27:27 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 11:27:27 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/usergrid/persistence/graph/GraphFig.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/784fe51c/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index ecb5b19..5968097 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -118,7 +118,7 @@ public interface GraphFig extends GuicyFig {
@Key(SHARD_CACHE_TIMEOUT)
long getShardCacheTimeout();
- @Default("60000")
+ @Default("90000")
@Key(SHARD_MIN_DELTA)
long getShardMinDelta();
[16/19] usergrid git commit: Fix it to be <=
Posted by mr...@apache.org.
Fix it to be <=
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be9bcb0b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be9bcb0b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be9bcb0b
Branch: refs/heads/master
Commit: be9bcb0b3f84f2ba09db22f7ea25c42762a44ddd
Parents: 1b1ed8b
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 12:23:44 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 12:23:44 2015 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/asyncevents/EventBuilderImpl.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be9bcb0b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index c9ae6da..48df8d0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -145,7 +145,7 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) < 0)
+ .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) <= 0)
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
[15/19] usergrid git commit: Use UUID compare for more precision.
Posted by mr...@apache.org.
Use UUID compare for more precision.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1b1ed8b6
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1b1ed8b6
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1b1ed8b6
Branch: refs/heads/master
Commit: 1b1ed8b63bbfd3c5fcf6f0e4a09f2009e8709d53
Parents: aea9264
Author: Michael Russo <mi...@gmail.com>
Authored: Thu Nov 19 12:21:48 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Thu Nov 19 12:21:48 2015 -0800
----------------------------------------------------------------------
.../usergrid/corepersistence/asyncevents/EventBuilderImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1b1ed8b6/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index f7f7ba3..c9ae6da 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.corepersistence.asyncevents;
import java.util.List;
+import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -144,7 +145,7 @@ public class EventBuilderImpl implements EventBuilder {
ecmDeleteObservable =
ecm.getVersions( entityId )
- .filter( mvccLogEntry-> mvccLogEntry.getVersion().timestamp() <= mostRecentlyMarked.getVersion().timestamp())
+ .filter( mvccLogEntry-> UUIDUtils.compare(mvccLogEntry.getVersion(), mostRecentlyMarked.getVersion()) < 0)
.buffer( serializationFig.getBufferSize() )
.doOnNext( buffer -> ecm.delete( buffer ) );
[09/19] usergrid git commit: Fixes the 2.5 checks in the test
Posted by mr...@apache.org.
Fixes the 2.5 checks in the test
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8ba0ef01
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8ba0ef01
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8ba0ef01
Branch: refs/heads/master
Commit: 8ba0ef01a90ac1352ea56203f07c042d73e24306
Parents: a33f616
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 17 11:19:06 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 17 11:19:06 2015 -0700
----------------------------------------------------------------------
.../serialization/impl/shard/NodeShardAllocationTest.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/8ba0ef01/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
index bc364cc..6671dec 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
@@ -84,7 +84,7 @@ public class NodeShardAllocationTest {
final long timeout = 30000;
when( graphFig.getShardCacheTimeout() ).thenReturn( timeout );
- when( graphFig.getShardMinDelta() ).thenReturn( timeout * 2 );
+ when( graphFig.getShardMinDelta() ).thenReturn( ( long ) (timeout * 2.5) );
}
@@ -111,7 +111,7 @@ public class NodeShardAllocationTest {
when( timeService.getCurrentTime() ).thenReturn( timeservicetime );
- final long expected = timeservicetime - 2 * graphFig.getShardCacheTimeout();
+ final long expected = ( long ) (timeservicetime - 2.5 * graphFig.getShardCacheTimeout());
final long returned = approximation.getMinTime();
@@ -722,7 +722,7 @@ public class NodeShardAllocationTest {
//now test something that passes.
- final long minDelta = cacheTimeout * 2;
+ final long minDelta = ( long ) (cacheTimeout * 2.5);
when( graphFig.getShardMinDelta() ).thenReturn( minDelta );
[18/19] usergrid git commit: formatting change only. This closes #442
Posted by mr...@apache.org.
formatting change only. This closes #442
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9bfbc531
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9bfbc531
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9bfbc531
Branch: refs/heads/master
Commit: 9bfbc531c4e70eedaff435033512e0d48f8df25c
Parents: e5b9730
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Nov 19 15:34:44 2015 -0500
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Nov 19 15:34:44 2015 -0500
----------------------------------------------------------------------
.../usergrid/persistence/model/entity/MapToEntityConverter.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9bfbc531/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
index a225820..aff7439 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/entity/MapToEntityConverter.java
@@ -47,7 +47,8 @@ public class MapToEntityConverter{
return fromMap( entity, map, null, null, topLevel );
}
- public Entity fromMap(final Entity entity,final Map<String, Object> map,final SchemaManager schemaManager, final String entityType, boolean topLevel) {
+ public Entity fromMap(final Entity entity,final Map<String, Object> map,final
+ SchemaManager schemaManager, final String entityType, boolean topLevel) {
for ( String fieldName : map.keySet() ) {