You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/08/28 05:36:52 UTC
[1/5] Updated OrderedMerge to use a faster implementation at runtime.
After initialization,
it's an O(1) emit operation as long as our produces are fast enough.
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-188 c69c1974a -> e040fdf4c
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/SimpleIdTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/SimpleIdTest.java b/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/SimpleIdTest.java
new file mode 100644
index 0000000..82a7b9a
--- /dev/null
+++ b/stack/corepersistence/model/src/test/java/org/apache/usergrid/persistence/model/entity/SimpleIdTest.java
@@ -0,0 +1,73 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.model.entity;
+
+
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import static org.junit.Assert.assertTrue;
+
+
+public class SimpleIdTest {
+
+ @Test
+ public void uuidComparison() {
+
+ final UUID firstId = UUIDGenerator.newTimeUUID();
+ final UUID secondId = UUIDGenerator.newTimeUUID();
+
+ final String type = "test";
+
+ SimpleId first = new SimpleId( firstId, type );
+ SimpleId second = new SimpleId( secondId, type );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( first.compareTo( first ) == 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+
+
+ @Test
+ public void typeComparison() {
+
+ final UUID uuid = UUIDGenerator.newTimeUUID();
+
+ final String firstType = "test1";
+ final String secondType = "test2";
+
+
+ SimpleId first = new SimpleId( uuid, firstType );
+ SimpleId second = new SimpleId( uuid, secondType );
+
+ assertTrue( first.compareTo( second ) < 0 );
+
+ assertTrue( first.compareTo( first ) == 0 );
+
+ assertTrue( second.compareTo( first ) > 0 );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index a9223f9..8080b4e 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -36,7 +36,7 @@
<antlr.version>3.4</antlr.version>
<archaius.version>0.5.12</archaius.version>
- <astyanax.version>1.56.49-SNAPSHOT-UG-1</astyanax.version>
+ <astyanax.version>1.56.49</astyanax.version>
<cassandra.version>1.2.15</cassandra.version>
<!-- <chop.version>1.0</chop.version>-->
<commons.codec.version>1.6</commons.codec.version>
[5/5] git commit: Updated OrderedMerge to use a faster implementation
at runtime. After initialization,
it's an O(1) emit operation as long as our produces are fast enough.
Posted by to...@apache.org.
Updated OrderedMerge to use a faster implementation at runtime. After initialization, it's an O(1) emit operation as long as our produces are fast enough.
Updated to fix comparator issues
Fixed tests
Changed algorithm. Counters not decrementing as expected on shard balancing.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e040fdf4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e040fdf4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e040fdf4
Branch: refs/heads/USERGRID-188
Commit: e040fdf4c2846188455b8cbcbac812e8a44ea69e
Parents: c69c197
Author: Todd Nine <to...@apache.org>
Authored: Mon Aug 18 17:19:11 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Wed Aug 27 21:35:49 2014 -0600
----------------------------------------------------------------------
.../src/test/resources/usergrid-UNIT.properties | 1 +
.../core/astyanax/ColumnNameIterator.java | 3 -
.../persistence/core/astyanax/ColumnSearch.java | 46 ++
.../astyanax/MultiKeyColumnNameIterator.java | 48 +-
.../core/astyanax/MultiRowColumnIterator.java | 251 +++++++++
.../persistence/core/rx/ObservableIterator.java | 2 +-
.../persistence/core/rx/OrderedMerge.java | 193 +++----
.../core/astyanax/ColumnNameIteratorTest.java | 205 ++++++++
.../MultiKeyColumnNameIteratorTest.java | 330 ++++++++++++
.../astyanax/MultiRowColumnIteratorTest.java | 387 ++++++++++++++
.../persistence/core/astyanax/TestUtils.java | 76 +++
.../core/cassandra/CassandraRule.java | 9 +
.../persistence/core/rx/OrderedMergeTest.java | 146 +++++-
.../common/src/test/resources/log4j.properties | 39 ++
.../src/test/resources/usergrid.properties | 1 +
.../usergrid/persistence/graph/GraphFig.java | 71 ++-
.../persistence/graph/SearchByEdge.java | 6 +
.../persistence/graph/SearchByEdgeType.java | 16 +
.../persistence/graph/guice/GraphModule.java | 4 +
.../persistence/graph/guice/MergedProxy.java | 34 --
.../graph/impl/SimpleMarkedEdge.java | 2 +-
.../graph/impl/SimpleSearchByEdge.java | 14 +-
.../graph/impl/SimpleSearchByEdgeType.java | 16 +-
.../graph/impl/SimpleSearchByIdType.java | 4 +-
.../graph/impl/stage/EdgeDeleteRepairImpl.java | 5 +-
.../graph/impl/stage/EdgeMetaRepairImpl.java | 7 +-
.../impl/stage/NodeDeleteListenerImpl.java | 7 +-
.../impl/shard/DirectedEdgeMeta.java | 189 ++++++-
.../graph/serialization/impl/shard/RowKey.java | 10 +
.../serialization/impl/shard/RowKeyType.java | 7 +-
.../graph/serialization/impl/shard/Shard.java | 3 +-
.../impl/shard/ShardEntryGroup.java | 62 ++-
.../impl/shard/ShardGroupCompaction.java | 31 +-
.../NodeShardCounterSerializationImpl.java | 9 +-
.../impl/shard/impl/EdgeRowKeySerializer.java | 63 ---
.../impl/shard/impl/EdgeSearcher.java | 79 ++-
.../impl/shard/impl/EdgeSerializer.java | 77 ---
.../shard/impl/EdgeShardRowKeySerializer.java | 103 ----
.../shard/impl/EdgeShardSerializationImpl.java | 1 +
.../shard/impl/NodeShardAllocationImpl.java | 85 ++-
.../impl/shard/impl/NodeShardCacheImpl.java | 4 +-
.../impl/shard/impl/RowSerializer.java | 63 ---
.../impl/shard/impl/RowTypeSerializer.java | 62 ---
.../shard/impl/ShardEntryGroupIterator.java | 39 +-
.../shard/impl/ShardGroupCompactionImpl.java | 519 +++++++++++++++----
.../impl/ShardedEdgeSerializationImpl.java | 187 +++----
.../impl/shard/impl/ShardsColumnIterator.java | 27 +-
.../shard/impl/SizebasedEdgeColumnFamilies.java | 4 +
.../DescendingTimestampComparator.java | 43 ++
.../DirectedEdgeDescendingComparator.java | 69 +++
.../impl/comparators/OrderedComparator.java | 52 ++
.../SourceDirectedEdgeDescendingComparator.java | 42 ++
.../TargetDirectedEdgeDescendingComparator.java | 42 ++
.../impl/serialize/EdgeRowKeySerializer.java | 63 +++
.../shard/impl/serialize/EdgeSerializer.java | 77 +++
.../serialize/EdgeShardRowKeySerializer.java | 103 ++++
.../shard/impl/serialize/RowSerializer.java | 63 +++
.../shard/impl/serialize/RowTypeSerializer.java | 62 +++
.../persistence/graph/GraphManagerLoadTest.java | 4 +-
.../graph/GraphManagerShardConsistencyIT.java | 372 +++++++++----
.../graph/GraphManagerStressTest.java | 6 +-
.../graph/impl/stage/EdgeDeleteRepairTest.java | 5 +-
.../impl/shard/NodeShardAllocationTest.java | 249 ++++++++-
.../impl/shard/ShardEntryGroupTest.java | 28 +-
.../impl/shard/ShardGroupCompactionTest.java | 226 ++++++++
.../shard/count/NodeShardApproximationTest.java | 12 +
.../shard/impl/ShardEntryGroupIteratorTest.java | 180 ++++---
...rceDirectedEdgeDescendingComparatorTest.java | 136 +++++
...getDirectedEdgeDescendingComparatorTest.java | 136 +++++
.../graph/test/util/EdgeTestUtils.java | 6 +-
.../graph/src/test/resources/log4j.properties | 1 +
.../src/test/resources/usergrid-UNIT.properties | 2 +
.../persistence/model/entity/SimpleIdTest.java | 73 +++
stack/corepersistence/pom.xml | 2 +-
74 files changed, 4510 insertions(+), 1091 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
index 9785b25..d038a4a 100644
--- a/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/collection/src/test/resources/usergrid-UNIT.properties
@@ -6,6 +6,7 @@ cassandra.hosts=localhost
cassandra.cluster_name=Usergrid
collections.keyspace=Usergrid_Collections
cassandra.timeout=5000
+cassandra.embedded=true
collections.keyspace.strategy.options=replication_factor:1
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIterator.java
index af4e1f9..6256e9c 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIterator.java
@@ -35,9 +35,6 @@ import com.netflix.hystrix.HystrixCommandGroupKey;
public class ColumnNameIterator<C, T> implements Iterable<T>, Iterator<T> {
- private static final HystrixCommandGroupKey GROUP_KEY = HystrixCommandGroupKey.Factory.asKey( "CassRead" );
-
-
private final RowQuery<?, C> rowQuery;
private final ColumnParser<C, T> parser;
private final boolean skipFirst;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnSearch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnSearch.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnSearch.java
new file mode 100644
index 0000000..589cb72
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ColumnSearch.java
@@ -0,0 +1,46 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import com.netflix.astyanax.util.RangeBuilder;
+
+
+/**
+ *
+ *
+ */
+public interface ColumnSearch<T> {
+
+ /**
+ * Set the start value supplied and the user supplied end value (if present)
+ * @param rangeBuilder
+ * @param value The value to set in the start
+ */
+ public void buildRange(final RangeBuilder rangeBuilder, final T value);
+
+ /**
+ * Set the range builder with the user supplied start and finish
+ * @param rangeBuilder
+ */
+ public void buildRange(final RangeBuilder rangeBuilder);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIterator.java
index c5a8c95..16ae97a 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIterator.java
@@ -24,22 +24,21 @@ package org.apache.usergrid.persistence.core.astyanax;
import java.util.Collection;
import java.util.Comparator;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.NoSuchElementException;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.Exchanger;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.Semaphore;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.rx.OrderedMerge;
import com.amazonaws.services.redshift.model.UnsupportedOptionException;
-import com.google.common.base.Preconditions;
+import rx.Notification;
import rx.Observable;
import rx.Subscriber;
+import rx.functions.Action1;
import rx.schedulers.Schedulers;
@@ -54,13 +53,26 @@ import rx.schedulers.Schedulers;
public class MultiKeyColumnNameIterator<C, T> implements Iterable<T>, Iterator<T> {
- private InnerIterator<T> iterator;
+ private static final Logger LOG = LoggerFactory.getLogger( MultiKeyColumnNameIterator.class );
+
+ private Iterator<T> iterator;
public MultiKeyColumnNameIterator( final Collection<ColumnNameIterator<C, T>> columnNameIterators,
final Comparator<T> comparator, final int bufferSize ) {
+ //optimization for single use case
+ if ( columnNameIterators.size() == 1 ) {
+ iterator = columnNameIterators.iterator().next();
+ return;
+ }
+
+
+ /**
+ * We have more than 1 iterator, subscribe to all of them on their own thread so they can
+ * produce in parallel. This way our inner iterator will be filled and processed the fastest
+ */
Observable<T>[] observables = new Observable[columnNameIterators.size()];
int i = 0;
@@ -77,9 +89,11 @@ public class MultiKeyColumnNameIterator<C, T> implements Iterable<T>, Iterator<T
Observable<T> merged = OrderedMerge.orderedMerge( comparator, bufferSize, observables ).distinctUntilChanged();
- iterator = new InnerIterator(bufferSize);
+ InnerIterator innerIterator = new InnerIterator( bufferSize );
+
+ merged.subscribe( innerIterator );
- merged.subscribe( iterator );
+ iterator = innerIterator;
}
@@ -114,9 +128,12 @@ public class MultiKeyColumnNameIterator<C, T> implements Iterable<T>, Iterator<T
*/
private final class InnerIterator<T> extends Subscriber<T> implements Iterator<T> {
- private CountDownLatch startLatch = new CountDownLatch( 1 );
+ private final CountDownLatch startLatch = new CountDownLatch( 1 );
- private final LinkedBlockingQueue<T> queue;
+ /**
+ * Use an ArrayBlockingQueue for faster access since our upper bounds is static
+ */
+ private final ArrayBlockingQueue<T> queue;
private Throwable error;
@@ -126,7 +143,7 @@ public class MultiKeyColumnNameIterator<C, T> implements Iterable<T>, Iterator<T
private InnerIterator( int maxSize ) {
- queue = new LinkedBlockingQueue<>( maxSize );
+ queue = new ArrayBlockingQueue<>( maxSize );
}
@@ -148,7 +165,6 @@ public class MultiKeyColumnNameIterator<C, T> implements Iterable<T>, Iterator<T
}
-
//this is almost a busy wait, and is intentional, if we have nothing to poll, we want to get it as soon
//as it's available. We generally only hit this once
do {
@@ -204,11 +220,13 @@ public class MultiKeyColumnNameIterator<C, T> implements Iterable<T>, Iterator<T
public void onNext( final T t ) {
//may block if we get full, that's expected behavior
+
try {
+ LOG.trace( "Received element {}" , t );
queue.put( t );
}
catch ( InterruptedException e ) {
- throw new RuntimeException( "Unable to take from queue" );
+ throw new RuntimeException( "Unable to insert to queue" );
}
startLatch.countDown();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
new file mode 100644
index 0000000..155ce84
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIterator.java
@@ -0,0 +1,251 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.TreeSet;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
+
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ColumnFamily;
+import com.netflix.astyanax.model.ColumnList;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.astyanax.model.Rows;
+import com.netflix.astyanax.query.RowSliceQuery;
+import com.netflix.astyanax.util.RangeBuilder;
+
+
+/**
+ *
+ *
+ */
+public class MultiRowColumnIterator<R, C, T> implements Iterator<T> {
+
+ private static final Logger LOG = LoggerFactory.getLogger( MultiRowColumnIterator.class );
+
+ private final int pageSize;
+
+ private final ColumnFamily<R, C> cf;
+
+
+ private final ColumnParser<C, T> columnParser;
+
+ private final ColumnSearch<T> columnSearch;
+
+ private final Comparator<T> comparator;
+
+
+ private final Collection<R> rowKeys;
+
+ private final Keyspace keyspace;
+
+ private final ConsistencyLevel consistencyLevel;
+
+
+ private T startColumn;
+
+ private boolean moreToFollow;
+
+ private Iterator<T> currentColumnIterator;
+
+
+ /**
+ * Remove after finding bug
+ */
+
+
+ // private int advanceCount;
+ //
+ // private final HashMap<T, SeekPosition> seenResults;
+
+ /**
+ * Complete Remove
+ */
+
+
+ /**
+ * Create the iterator
+ */
+ public MultiRowColumnIterator( final Keyspace keyspace, final ColumnFamily<R, C> cf,
+ final ConsistencyLevel consistencyLevel, final ColumnParser<C, T> columnParser,
+ final ColumnSearch<T> columnSearch, final Comparator<T> comparator,
+ final Collection<R> rowKeys, final int pageSize ) {
+ this.cf = cf;
+ this.pageSize = pageSize;
+ this.columnParser = columnParser;
+ this.columnSearch = columnSearch;
+ this.comparator = comparator;
+ this.rowKeys = rowKeys;
+ this.keyspace = keyspace;
+ this.consistencyLevel = consistencyLevel;
+ this.moreToFollow = false;
+
+ // seenResults = new HashMap<>( pageSize * 10 );
+ }
+
+
+ @Override
+ public boolean hasNext() {
+
+ if ( currentColumnIterator == null || ( !currentColumnIterator.hasNext() && moreToFollow ) ) {
+ advance();
+ }
+
+
+ return currentColumnIterator.hasNext();
+ }
+
+
+ @Override
+ public T next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException( "No new element exists" );
+ }
+
+ final T next = currentColumnIterator.next();
+
+ // LOG.trace( "Emitting {}", next );
+
+ return next;
+ }
+
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException( "Remove is unsupported this is a read only iterator" );
+ }
+
+
+ public void advance() {
+
+
+ /**
+ * If the edge is present, we need to being seeking from this
+ */
+
+
+ //TODO, finalize why this isn't working as expected
+ final int selectSize = startColumn == null ? pageSize : pageSize + 1;
+
+ final RangeBuilder rangeBuilder = new RangeBuilder();
+
+
+ //set the range into the search
+
+ if ( startColumn == null ) {
+ columnSearch.buildRange( rangeBuilder );
+ }
+ else {
+ columnSearch.buildRange( rangeBuilder, startColumn );
+ }
+
+
+ rangeBuilder.setLimit( selectSize );
+
+
+ /**
+ * Get our list of slices
+ */
+ final RowSliceQuery<R, C> query =
+ keyspace.prepareQuery( cf ).setConsistencyLevel( consistencyLevel ).getKeySlice( rowKeys )
+ .withColumnRange( rangeBuilder.build() );
+
+ final Rows<R, C> result = HystrixCassandra.user( query ).getResult();
+
+ final TreeSet<T> mergedResults = new TreeSet<>( comparator );
+
+
+ //now aggregate them together
+
+ for ( final R key : result.getKeys() ) {
+ final ColumnList<C> columns = result.getRow( key ).getColumns();
+ final int size = columns.size();
+
+ int readIndex = 0;
+
+ //skip the first since it's equal and has been set
+ if ( startColumn != null && size > 0 ) {
+ final T returnedValue = columnParser.parseColumn( columns.getColumnByIndex( 0 ) );
+
+ if ( comparator.compare( returnedValue, startColumn ) == 0 ) {
+ readIndex++;
+ }
+ }
+
+
+// T previous = null;
+
+ for (; readIndex < size; readIndex++ ) {
+ final Column<C> column = columns.getColumnByIndex( readIndex );
+ final T returnedValue = columnParser.parseColumn( column );
+
+ /**
+ * DO NOT remove this section of code. If you're seeing inconsistent results during shard transition, you'll
+ * need to enable this
+ */
+//
+// if ( previous != null && comparator.compare( previous, returnedValue ) == 0 ) {
+// throw new RuntimeException( String.format(
+// "Cassandra returned 2 unique columns, but your comparator marked them as equal. This " +
+// "indicates a bug in your comparator. Previous value was %s and current value is " +
+// "%s",
+// previous, returnedValue ) );
+// }
+//
+// previous = returnedValue;
+
+ mergedResults.add( returnedValue );
+
+ //prune the mergedResults
+ while ( mergedResults.size() > pageSize ) {
+ mergedResults.pollLast();
+ }
+ }
+
+ LOG.trace( "Read {} columns from row key {}", readIndex, key );
+ LOG.trace( "Candidate result set size is {}", mergedResults.size() );
+ }
+
+
+ //we've parsed everything truncate to the first pageSize, it's all we can ensure is correct without another
+ //trip back to cassandra
+
+ startColumn = mergedResults.last();
+
+ moreToFollow = mergedResults.size() == pageSize;
+
+ currentColumnIterator = mergedResults.iterator();
+
+ LOG.trace( "Finished parsing {} rows for a total of {} results", rowKeys.size(), mergedResults.size() );
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
index a6c3aa9..2bd1edb 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/ObservableIterator.java
@@ -64,7 +64,7 @@ public abstract class ObservableIterator<T> implements Observable.OnSubscribe<T>
while ( itr.hasNext() && !subscriber.isUnsubscribed() ) {
final T next = itr.next();
- log.trace( "Iterator '{}' emitting item '{}'", name, next );
+// log.trace( "Iterator '{}' emitting item '{}'", name, next );
subscriber.onNext( next );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
index 4032176..cdad0d1 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/rx/OrderedMerge.java
@@ -20,17 +20,21 @@
package org.apache.usergrid.persistence.core.rx;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.Deque;
import java.util.LinkedList;
import java.util.List;
+import java.util.NavigableSet;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.collect.TreeMultimap;
+
import rx.Observable;
import rx.Subscriber;
import rx.Subscription;
@@ -40,9 +44,9 @@ import rx.subscriptions.CompositeSubscription;
/**
* Produces a single Observable from multiple ordered source observables. The same as the "merge" step in a merge sort.
- * Ensure that your comparator matches the ordering of your inputs, or you may get strange results.
- * The current implementation requires each Observable to be running in it's own thread. Once backpressure in RX is
- * implemented, this requirement can be removed.
+ * Ensure that your comparator matches the ordering of your inputs, or you may get strange results. The current
+ * implementation requires each Observable to be running in it's own thread. Once backpressure in RX is implemented,
+ * this requirement can be removed.
*/
public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
@@ -74,7 +78,7 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
//when a subscription is received, we need to subscribe on each observable
- SubscriberCoordinator coordinator = new SubscriberCoordinator( comparator, outerOperation );
+ SubscriberCoordinator coordinator = new SubscriberCoordinator( comparator, outerOperation, observables.length );
InnerObserver<T>[] innerObservers = new InnerObserver[observables.length];
@@ -84,7 +88,7 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
for ( int i = 0; i < observables.length; i++ ) {
//subscribe to each one and add it to the composite
//create a new inner and subscribe
- final InnerObserver<T> inner = new InnerObserver<T>( coordinator, maxBufferSize );
+ final InnerObserver<T> inner = new InnerObserver<T>( coordinator, maxBufferSize, i );
coordinator.add( inner );
@@ -117,16 +121,19 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
private volatile boolean readyToProduce = false;
- private final Comparator<T> comparator;
private final Subscriber<? super T> subscriber;
+ private final TreeMultimap<T, InnerObserver<T>> nextValues;
private final List<InnerObserver<T>> innerSubscribers;
+ private final ArrayDeque<InnerObserver<T>> toProduce;
- private SubscriberCoordinator( final Comparator<T> comparator, final Subscriber<? super T> subscriber ) {
+ private SubscriberCoordinator( final Comparator<T> comparator, final Subscriber<? super T> subscriber,
+ final int innerSize ) {
//we only want to emit events serially
this.subscriber = new SerializedSubscriber( subscriber );
- this.innerSubscribers = new ArrayList<InnerObserver<T>>();
- this.comparator = comparator;
+ this.innerSubscribers = new ArrayList<>( innerSize );
+ this.nextValues = TreeMultimap.create( comparator, InnerObserverComparator.INSTANCE );
+ this.toProduce = new ArrayDeque<>( innerSize );
}
@@ -146,7 +153,7 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
log.trace( "Completing Observable. Draining elements from the subscribers", innerSubscribers.size() );
//Drain the queues
- while ( !subscriber.isUnsubscribed() && !drained() ) {
+ while ( !subscriber.isUnsubscribed() && (!nextValues.isEmpty() || !toProduce.isEmpty()) ) {
next();
}
@@ -158,6 +165,7 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
public void add( InnerObserver<T> inner ) {
this.innerSubscribers.add( inner );
+ this.toProduce.add( inner );
}
@@ -168,100 +176,108 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
public void next() {
- //nothing to do, we haven't started emitting values yet
- if ( !readyToProduce() ) {
- return;
- }
-
//we want to emit items in order, so we synchronize our next
synchronized ( this ) {
+ /**
+ * Init before our loop
+ */
+ while ( !toProduce.isEmpty() ) {
+
+ InnerObserver<T> inner = toProduce.pop();
+
+ //This has nothing left to produce, skip it
+ if ( inner.drained ) {
+ continue;
+ }
+
+ final T nextKey = inner.peek();
+
+ //we can't produce, not everything has an element to inspect, leave it in the set to produce next
+ // time
+ if ( nextKey == null ) {
+ toProduce.push( inner );
+ return;
+ }
+
+ //add it to our fast access set
+ nextValues.put( nextKey, inner );
+ }
+
+
+ //take as many elements as we can until we hit a case where we can't take anymore
+ while ( !nextValues.isEmpty() ) {
- //take as many elements as we can until we hit the completed case
- while ( true ) {
- InnerObserver<T> maxObserver = null;
- T max = null;
/**
- * TODO T.N. change this to be an 0(1) for min and O(log n) to update after pop rather than O(n*inner)
+ * Get our lowest key and begin producing until we can't produce any longer
*/
- for ( InnerObserver<T> inner : innerSubscribers ) {
+ final T lowestKey = nextValues.keySet().first();
- //nothing to do, this inner
- //we're done skip it
- if ( inner.drained ) {
- continue;
- }
+ //we need to create a copy, otherwise we receive errors. We use ArrayDque
+ NavigableSet<InnerObserver<T>> nextObservers = nextValues.get( lowestKey );
- final T current = inner.peek();
+ while ( !nextObservers.isEmpty() ) {
- /**
- * Our current is null but we're not drained (I.E we haven't finished and completed consuming)
- * This means the producer is slow, and we don't have a complete set to compare,
- * we can't produce. Bail and try again on the next event.
- */
- if ( current == null ) {
- return;
- }
+ final InnerObserver<T> inner = nextObservers.pollFirst();
+ nextValues.remove( lowestKey, inner );
- if ( max == null || ( current != null
- && comparator.compare( current, max ) > 0 ) ) {
- maxObserver = inner;
- max = current;
- }
- }
+ final T value = inner.pop();
- //No max observer was ever assigned, meaning all our inners are drained, break from loop
- if ( maxObserver == null ) {
- return;
- }
+ log.trace( "Emitting value {}", value );
- log.trace( "Max element is item {}", max );
+ subscriber.onNext( value );
- subscriber.onNext( maxObserver.pop() );
- }
- }
- }
+ final T nextKey = inner.peek();
+ //nothing to peek, it's either drained or slow
+ if ( nextKey == null ) {
- /**
- * Return true if we're ready to produce
- */
- private boolean readyToProduce() {
- if ( readyToProduce ) {
- return true;
- }
+ //it's drained, nothing left to do
+ if ( inner.drained ) {
+ continue;
+ }
+ //it's slow, we can't process because we don't know if this is another min value without
+ // inspecting it. Stop emitting and try again next pass through
+ toProduce.push( inner );
+ return;
+ }
- //perform an audit
- for ( InnerObserver<T> inner : innerSubscribers ) {
- if ( !inner.started ) {
- readyToProduce = false;
- return false;
+ //we have a next value, insert it and keep running
+ nextValues.put( nextKey, inner );
+ }
}
}
+ }
- readyToProduce = true;
- //we'll try again next time
- return false;
- }
+// /**
+// * Return true if every inner observer has been drained
+// */
+// private boolean drained() {
+// //perform an audit
+// for ( InnerObserver<T> inner : innerSubscribers ) {
+// if ( !inner.drained ) {
+// return false;
+// }
+// }
+//
+// return true;
+// }
+ }
- /**
- * Return true if every inner observer has been drained
- */
- private boolean drained() {
- //perform an audit
- for ( InnerObserver<T> inner : innerSubscribers ) {
- if ( !inner.drained ) {
- return false;
- }
- }
+ private static final class InnerObserverComparator implements Comparator<InnerObserver> {
- return true;
+ private static final InnerObserverComparator INSTANCE = new InnerObserverComparator();
+
+
+ @Override
+ public int compare( final InnerObserver o1, final InnerObserver o2 ) {
+ return Integer.compare( o1.id, o2.id );
}
}
@@ -269,13 +285,18 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
private static final class InnerObserver<T> extends Subscriber<T> {
private final SubscriberCoordinator<T> coordinator;
- private final Deque<T> items = new LinkedList<T>();
+ private final Deque<T> items = new LinkedList<>();
private final int maxQueueSize;
/**
* TODO: T.N. Once backpressure makes it into RX Java, this needs to be remove and should use backpressure
*/
private final Semaphore semaphore;
+ /**
+ * Our id so we have something unique to compare in the multimap
+ */
+ public final int id;
+
/**
* Flags for synchronization with coordinator. Multiple threads may be used, so volatile is required
@@ -285,9 +306,11 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
private volatile boolean drained = false;
- public InnerObserver( final SubscriberCoordinator<T> coordinator, final int maxQueueSize ) {
+ public InnerObserver( final SubscriberCoordinator<T> coordinator, final int maxQueueSize, final int id ) {
this.coordinator = coordinator;
this.maxQueueSize = maxQueueSize;
+ this.id = id;
+
this.semaphore = new Semaphore( maxQueueSize );
}
@@ -302,8 +325,7 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
* release this semaphore and invoke next. Both these calls can be removed when backpressure is added.
* We need the next to force removal of other inner consumers
*/
- coordinator.onCompleted();
-
+ coordinator.onCompleted();
}
@@ -316,21 +338,14 @@ public final class OrderedMerge<T> implements Observable.OnSubscribe<T> {
@Override
public void onNext( T a ) {
- log.trace( "Received {}", a );
-
try {
this.semaphore.acquire();
}
catch ( InterruptedException e ) {
- onError(e);
- }
-
- if ( items.size() == maxQueueSize ) {
- RuntimeException e =
- new RuntimeException( "The maximum queue size of " + maxQueueSize + " has been reached" );
onError( e );
}
+
items.add( a );
started = true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
new file mode 100644
index 0000000..aabab24
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
@@ -0,0 +1,205 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.util.HashMap;
+
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ColumnFamily;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.astyanax.query.RowQuery;
+import com.netflix.astyanax.serializers.LongSerializer;
+import com.netflix.astyanax.serializers.StringSerializer;
+import com.netflix.astyanax.util.RangeBuilder;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class ColumnNameIteratorTest {
+
+
+ @ClassRule
+ public static CassandraRule rule = new CassandraRule();
+
+ protected static Keyspace keyspace;
+
+ protected ApplicationScope scope;
+
+ protected static ColumnFamily<String, Long> COLUMN_FAMILY =
+ new ColumnFamily<>( "LongTests", StringSerializer.get(), LongSerializer.get() );
+
+ protected static final boolean TRUE = true;
+
+
+ @BeforeClass
+ public static void setup() throws ConnectionException {
+
+
+ final CassandraConfig cassandraConfig = new CassandraConfig() {
+ @Override
+ public ConsistencyLevel getReadCL() {
+ return ConsistencyLevel.CL_QUORUM;
+ }
+
+
+ @Override
+ public ConsistencyLevel getWriteCL() {
+ return ConsistencyLevel.CL_QUORUM;
+ }
+ };
+
+
+ AstyanaxKeyspaceProvider astyanaxKeyspaceProvider =
+ new AstyanaxKeyspaceProvider( rule.getCassandraFig(), cassandraConfig );
+
+ keyspace = astyanaxKeyspaceProvider.get();
+
+ TestUtils.createKeyspace( keyspace );
+
+ TestUtils.createColumnFamiliy( keyspace, COLUMN_FAMILY, new HashMap<String, Object>() );
+ }
+
+
+ @Test
+ public void testSingleIterator() {
+
+ String rowKey1 = UUIDGenerator.newTimeUUID().toString();
+
+
+ final long maxValue = 10000;
+
+
+ /**
+ * Write to both rows in parallel
+ */
+
+
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ batch.withRow( COLUMN_FAMILY, rowKey1 ).putColumn( i, TRUE );
+
+ if ( i % 1000 == 0 ) {
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+
+
+ //now read from them, we should get an iterator that repeats from 0 to 9999 2 x for every entry
+
+ final ColumnParser<Long, Long> longParser = new ColumnParser<Long, Long>() {
+ @Override
+ public Long parseColumn( final Column<Long> column ) {
+ return column.getName();
+ }
+ };
+
+
+ //ensure we have to make several trips, purposefully set to a nonsensical value to ensure we make all the
+ // trips required
+ final RangeBuilder forwardRange = new RangeBuilder().setLimit( 720 );
+
+
+ final RowQuery<String, Long> forwardQuery =
+ keyspace.prepareQuery( COLUMN_FAMILY ).getKey( rowKey1 ).withColumnRange( forwardRange.build() );
+
+
+ ColumnNameIterator<Long, Long> itr = new ColumnNameIterator<>( forwardQuery, longParser, false );
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ assertEquals( i, itr.next().longValue() );
+ }
+
+ //now test it in reverse
+
+
+ final RangeBuilder reverseRange = new RangeBuilder().setLimit( 720 ).setReversed( true );
+
+
+ final RowQuery<String, Long> reverseQuery =
+ keyspace.prepareQuery( COLUMN_FAMILY ).getKey( rowKey1 ).withColumnRange( reverseRange.build() );
+
+
+ ColumnNameIterator<Long, Long> reverseItr = new ColumnNameIterator<>( reverseQuery, longParser, false );
+
+ for ( long i = maxValue - 1; i > -1; i-- ) {
+ assertEquals( i, reverseItr.next().longValue() );
+ }
+ }
+
+
+ // /**
+ // * Write to both rows in parallel
+ // */
+ // Observable.from( new String[]{rowKey1, rowKey2} ).parallel( new Func1<Observable<String>,
+ // Observable<String>>() {
+ // @Override
+ // public Observable<String> call( final Observable<String> stringObservable ) {
+ // return stringObservable.doOnNext( new Action1<String>() {
+ // @Override
+ // public void call( final String key ) {
+ //
+ // final MutationBatch batch = keyspace.prepareMutationBatch();
+ //
+ // for(long i = 0; i < maxValue; i ++){
+ // batch.withRow( COLUMN_FAMILY, key).putColumn( i, TRUE );
+ //
+ // if(i % 1000 == 0){
+ // try {
+ // batch.execute();
+ // }
+ // catch ( ConnectionException e ) {
+ // throw new RuntimeException(e);
+ // }
+ // }
+ //
+ // }
+ //
+ // }
+ // } );
+ // }
+ // } ).toBlocking().last();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
new file mode 100644
index 0000000..6762588
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
@@ -0,0 +1,330 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ColumnFamily;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.astyanax.query.RowQuery;
+import com.netflix.astyanax.serializers.LongSerializer;
+import com.netflix.astyanax.serializers.StringSerializer;
+import com.netflix.astyanax.util.RangeBuilder;
+
+import rx.Observable;
+import rx.functions.Action1;
+import rx.functions.Func1;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class MultiKeyColumnNameIteratorTest {
+
+ @ClassRule
+ public static CassandraRule rule = new CassandraRule();
+
+ protected static Keyspace keyspace;
+
+ protected ApplicationScope scope;
+
+ protected static ColumnFamily<String, Long> COLUMN_FAMILY =
+ new ColumnFamily<>( "MultiKeyLongTests", StringSerializer.get(), LongSerializer.get() );
+
+ protected static final boolean TRUE = true;
+
+
+ @BeforeClass
+ public static void setup() throws ConnectionException {
+
+
+ final CassandraConfig cassandraConfig = new CassandraConfig() {
+ @Override
+ public ConsistencyLevel getReadCL() {
+ return ConsistencyLevel.CL_QUORUM;
+ }
+
+
+ @Override
+ public ConsistencyLevel getWriteCL() {
+ return ConsistencyLevel.CL_QUORUM;
+ }
+ };
+
+
+ AstyanaxKeyspaceProvider astyanaxKeyspaceProvider =
+ new AstyanaxKeyspaceProvider( rule.getCassandraFig(), cassandraConfig );
+
+ keyspace = astyanaxKeyspaceProvider.get();
+
+ TestUtils.createKeyspace( keyspace );
+
+ TestUtils.createColumnFamiliy( keyspace, COLUMN_FAMILY, new HashMap<String, Object>() );
+ }
+
+
+ @Test
+ public void multiIterator() {
+
+ final String rowKey1 = UUIDGenerator.newTimeUUID().toString();
+
+ final String rowKey2 = UUIDGenerator.newTimeUUID().toString();
+
+ final String rowKey3 = UUIDGenerator.newTimeUUID().toString();
+
+
+ final long maxValue = 10000;
+
+ /**
+ * Write to both rows in parallel
+ */
+ Observable.from( new String[] { rowKey1, rowKey2, rowKey3 } )
+ .parallel( new Func1<Observable<String>, Observable<String>>() {
+ @Override
+ public Observable<String> call( final Observable<String> stringObservable ) {
+ return stringObservable.doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String key ) {
+
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ batch.withRow( COLUMN_FAMILY, key ).putColumn( i, TRUE );
+
+ if ( i % 1000 == 0 ) {
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ } );
+ }
+ } ).toBlocking().last();
+
+
+ //create 3 iterators
+
+ ColumnNameIterator<Long, Long> row1Iterator = createIterator( rowKey1, false );
+ ColumnNameIterator<Long, Long> row2Iterator = createIterator( rowKey2, false );
+ ColumnNameIterator<Long, Long> row3Iterator = createIterator( rowKey3, false );
+
+ final Comparator<Long> ascendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return Long.compare( o1, o2 );
+ }
+ };
+
+ /**
+ * Again, arbitrary buffer size to attempt we buffer at some point
+ */
+ final MultiKeyColumnNameIterator<Long, Long> ascendingItr =
+ new MultiKeyColumnNameIterator<>( Arrays.asList( row1Iterator, row2Iterator, row3Iterator ),
+ ascendingComparator, 900 );
+
+
+ //ensure we have to make several trips, purposefully set to a nonsensical value to ensure we make all the
+ // trips required
+
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ assertEquals( i, ascendingItr.next().longValue() );
+ }
+
+ //now test it in reverse
+
+ ColumnNameIterator<Long, Long> row1IteratorDesc = createIterator( rowKey1, true );
+ ColumnNameIterator<Long, Long> row2IteratorDesc = createIterator( rowKey2, true );
+ ColumnNameIterator<Long, Long> row3IteratorDesc = createIterator( rowKey3, true );
+
+ final Comparator<Long> descendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return ascendingComparator.compare( o1, o2 ) * -1;
+ }
+ };
+
+ /**
+ * Again, arbitrary buffer size to attempt we buffer at some point
+ */
+ final MultiKeyColumnNameIterator<Long, Long> descendingItr =
+ new MultiKeyColumnNameIterator<>( Arrays.asList( row1IteratorDesc, row2IteratorDesc, row3IteratorDesc ),
+ descendingComparator, 900 );
+
+
+ for ( long i = maxValue - 1; i > -1; i-- ) {
+ assertEquals( i, descendingItr.next().longValue() );
+ }
+ }
+
+
+ @Test
+ public void singleIterator() {
+
+ final String rowKey1 = UUIDGenerator.newTimeUUID().toString();
+
+
+
+ final long maxValue = 10000;
+
+ /**
+ * Write to both rows in parallel
+ */
+ Observable.just( rowKey1 )
+ .parallel( new Func1<Observable<String>, Observable<String>>() {
+ @Override
+ public Observable<String> call( final Observable<String> stringObservable ) {
+ return stringObservable.doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String key ) {
+
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ batch.withRow( COLUMN_FAMILY, key ).putColumn( i, TRUE );
+
+ if ( i % 1000 == 0 ) {
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ } );
+ }
+ } ).toBlocking().last();
+
+
+ //create 3 iterators
+
+ ColumnNameIterator<Long, Long> row1Iterator = createIterator( rowKey1, false );
+
+ final Comparator<Long> ascendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return Long.compare( o1, o2 );
+ }
+ };
+
+ /**
+ * Again, arbitrary buffer size to attempt we buffer at some point
+ */
+ final MultiKeyColumnNameIterator<Long, Long> ascendingItr =
+ new MultiKeyColumnNameIterator<>( Arrays.asList( row1Iterator ),
+ ascendingComparator, 900 );
+
+
+ //ensure we have to make several trips, purposefully set to a nonsensical value to ensure we make all the
+ // trips required
+
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ //we have 3 iterators, so we should get each value 3 times in the aggregation
+ assertEquals( i, ascendingItr.next().longValue() );
+ }
+
+ //now test it in reverse
+
+ ColumnNameIterator<Long, Long> row1IteratorDesc = createIterator( rowKey1, true );
+
+ final Comparator<Long> descendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return ascendingComparator.compare( o1, o2 ) * -1;
+ }
+ };
+
+ /**
+ * Again, arbitrary buffer size to attempt we buffer at some point
+ */
+ final MultiKeyColumnNameIterator<Long, Long> descendingItr =
+ new MultiKeyColumnNameIterator<>( Arrays.asList( row1IteratorDesc),
+ descendingComparator, 900 );
+
+
+ for ( long i = maxValue - 1; i > -1; i-- ) {
+ assertEquals( i, descendingItr.next().longValue() );
+ }
+ }
+
+
+ private static ColumnNameIterator<Long, Long> createIterator( final String rowKey, final boolean reversed ) {
+
+
+ final ColumnParser<Long, Long> longParser = new ColumnParser<Long, Long>() {
+ @Override
+ public Long parseColumn( final Column<Long> column ) {
+ return column.getName();
+ }
+ };
+
+ final RangeBuilder forwardRange = new RangeBuilder().setLimit( 720 ).setReversed( reversed );
+
+
+ final RowQuery<String, Long> forwardQuery =
+ keyspace.prepareQuery( COLUMN_FAMILY ).getKey( rowKey ).withColumnRange( forwardRange.build() );
+
+
+ ColumnNameIterator<Long, Long> itr = new ColumnNameIterator<>( forwardQuery, longParser, false );
+
+ return itr;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
new file mode 100644
index 0000000..8a8d0b0
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
@@ -0,0 +1,387 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.concurrent.CountDownLatch;
+
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.model.Column;
+import com.netflix.astyanax.model.ColumnFamily;
+import com.netflix.astyanax.model.ConsistencyLevel;
+import com.netflix.astyanax.serializers.LongSerializer;
+import com.netflix.astyanax.serializers.StringSerializer;
+import com.netflix.astyanax.util.RangeBuilder;
+
+import rx.Observable;
+import rx.Observer;
+import rx.Subscription;
+import rx.functions.Action1;
+import rx.functions.Func1;
+
+import static org.junit.Assert.assertEquals;
+
+
+public class MultiRowColumnIteratorTest {
+
+ @ClassRule
+ public static CassandraRule rule = new CassandraRule();
+
+ protected static Keyspace keyspace;
+
+ protected ApplicationScope scope;
+
+ protected static ColumnFamily<String, Long> COLUMN_FAMILY =
+ new ColumnFamily<>( "MultiRowLongTests", StringSerializer.get(), LongSerializer.get() );
+
+ protected static final boolean TRUE = true;
+
+
+ @BeforeClass
+ public static void setup() throws ConnectionException {
+
+
+ final CassandraConfig cassandraConfig = new CassandraConfig() {
+ @Override
+ public ConsistencyLevel getReadCL() {
+ return ConsistencyLevel.CL_QUORUM;
+ }
+
+
+ @Override
+ public ConsistencyLevel getWriteCL() {
+ return ConsistencyLevel.CL_QUORUM;
+ }
+ };
+
+
+ AstyanaxKeyspaceProvider astyanaxKeyspaceProvider =
+ new AstyanaxKeyspaceProvider( rule.getCassandraFig(), cassandraConfig );
+
+ keyspace = astyanaxKeyspaceProvider.get();
+
+ TestUtils.createKeyspace( keyspace );
+
+ TestUtils.createColumnFamiliy( keyspace, COLUMN_FAMILY, new HashMap<String, Object>() );
+ }
+
+
+ @Test
+ public void multiIterator() throws InterruptedException {
+
+ final String rowKey1 = UUIDGenerator.newTimeUUID().toString();
+
+ final String rowKey2 = UUIDGenerator.newTimeUUID().toString();
+
+ final String rowKey3 = UUIDGenerator.newTimeUUID().toString();
+
+
+ final long maxValue = 10000;
+
+ final CountDownLatch latch = new CountDownLatch( 3 );
+
+
+ writeData( latch, rowKey1, maxValue, 1 );
+ writeData( latch, rowKey2, maxValue, 2 );
+ writeData( latch, rowKey3, maxValue, 10 );
+
+
+ latch.await();
+
+
+ //create 3 iterators
+
+
+ final ColumnParser<Long, Long> longParser = new ColumnParser<Long, Long>() {
+ @Override
+ public Long parseColumn( final Column<Long> column ) {
+ return column.getName();
+ }
+ };
+
+
+ final ColumnSearch<Long> ascendingSearch = new ColumnSearch<Long>() {
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder, final Long value ) {
+ rangeBuilder.setStart( value );
+ }
+
+
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder ) {
+
+ }
+ };
+
+
+ final Comparator<Long> ascendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return Long.compare( o1, o2 );
+ }
+ };
+
+
+ final Collection<String> rowKeys = Arrays.asList( rowKey1, rowKey2, rowKey3 );
+
+ MultiRowColumnIterator<String, Long, Long> ascendingItr =
+ new MultiRowColumnIterator<>( keyspace, COLUMN_FAMILY, ConsistencyLevel.CL_QUORUM, longParser,
+ ascendingSearch, ascendingComparator, rowKeys, 852 );
+
+
+ //ensure we have to make several trips, purposefully set to a nonsensical value to ensure we make all the
+ // trips required
+
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ assertEquals( i, ascendingItr.next().longValue() );
+ }
+
+
+ final ColumnSearch<Long> descendingSearch = new ColumnSearch<Long>() {
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder, final Long value ) {
+ rangeBuilder.setStart( value );
+ buildRange( rangeBuilder );
+ }
+
+
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder ) {
+ rangeBuilder.setReversed( true );
+ }
+ };
+
+
+ final Comparator<Long> descendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return ascendingComparator.compare( o1, o2 ) * -1;
+ }
+ };
+
+
+ MultiRowColumnIterator<String, Long, Long> descendingItr =
+ new MultiRowColumnIterator<>( keyspace, COLUMN_FAMILY, ConsistencyLevel.CL_QUORUM, longParser,
+ descendingSearch, descendingComparator, rowKeys, 712 );
+
+ for ( long i = maxValue - 1; i > -1; i-- ) {
+ assertEquals( i, descendingItr.next().longValue() );
+ }
+ }
+
+
+ @Test
+ public void singleIterator() {
+
+ final String rowKey1 = UUIDGenerator.newTimeUUID().toString();
+
+
+ final long maxValue = 10000;
+
+ /**
+ * Write to both rows in parallel
+ */
+ Observable.just( rowKey1 ).parallel( new Func1<Observable<String>, Observable<String>>() {
+ @Override
+ public Observable<String> call( final Observable<String> stringObservable ) {
+ return stringObservable.doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String key ) {
+
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ batch.withRow( COLUMN_FAMILY, key ).putColumn( i, TRUE );
+
+ if ( i % 1000 == 0 ) {
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ } );
+ }
+ } ).toBlocking().last();
+
+
+ //create 3 iterators
+
+ final ColumnParser<Long, Long> longParser = new ColumnParser<Long, Long>() {
+ @Override
+ public Long parseColumn( final Column<Long> column ) {
+ return column.getName();
+ }
+ };
+
+
+ final ColumnSearch<Long> ascendingSearch = new ColumnSearch<Long>() {
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder, final Long value ) {
+ rangeBuilder.setStart( value );
+ }
+
+
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder ) {
+
+ }
+ };
+
+
+ final Comparator<Long> ascendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return Long.compare( o1, o2 );
+ }
+ };
+
+
+ final Collection<String> rowKeys = Arrays.asList( rowKey1 );
+
+ MultiRowColumnIterator<String, Long, Long> ascendingItr =
+ new MultiRowColumnIterator<>( keyspace, COLUMN_FAMILY, ConsistencyLevel.CL_QUORUM, longParser,
+ ascendingSearch, ascendingComparator, rowKeys, 712 );
+
+
+ //ensure we have to make several trips, purposefully set to a nonsensical value to ensure we make all the
+ // trips required
+
+
+ for ( long i = 0; i < maxValue; i++ ) {
+ assertEquals( i, ascendingItr.next().longValue() );
+ }
+
+
+ final ColumnSearch<Long> descendingSearch = new ColumnSearch<Long>() {
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder, final Long value ) {
+ rangeBuilder.setStart( value );
+ buildRange( rangeBuilder );
+ }
+
+
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder ) {
+ rangeBuilder.setReversed( true );
+ }
+ };
+
+
+ final Comparator<Long> descendingComparator = new Comparator<Long>() {
+
+ @Override
+ public int compare( final Long o1, final Long o2 ) {
+ return ascendingComparator.compare( o1, o2 ) * -1;
+ }
+ };
+
+
+ MultiRowColumnIterator<String, Long, Long> descendingItr =
+ new MultiRowColumnIterator<>( keyspace, COLUMN_FAMILY, ConsistencyLevel.CL_QUORUM, longParser,
+ descendingSearch, descendingComparator, rowKeys, 712 );
+
+ for ( long i = maxValue - 1; i > -1; i-- ) {
+ assertEquals( i, descendingItr.next().longValue() );
+ }
+ }
+
+
+ private void writeData(final CountDownLatch latch, final String rowKey, final long maxValue, final long mod){
+
+ Observable.just( rowKey ).doOnNext( new Action1<String>() {
+ @Override
+ public void call( final String key ) {
+
+ final MutationBatch batch = keyspace.prepareMutationBatch();
+
+ for ( long i = 0; i < maxValue; i++ ) {
+
+ if ( i % mod == 0 ) {
+ batch.withRow( COLUMN_FAMILY, key ).putColumn( i, TRUE );
+ }
+
+ if ( i % 1000 == 0 ) {
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ }
+
+ try {
+ batch.execute();
+ }
+ catch ( ConnectionException e ) {
+ throw new RuntimeException( e );
+ }
+ }
+ } ).subscribe( new Observer<String>() {
+ @Override
+ public void onCompleted() {
+ latch.countDown();
+ }
+
+
+ @Override
+ public void onError( final Throwable e ) {
+
+ }
+
+
+ @Override
+ public void onNext( final String s ) {
+
+ }
+ } );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/TestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/TestUtils.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/TestUtils.java
new file mode 100644
index 0000000..1ede643
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/TestUtils.java
@@ -0,0 +1,76 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+import com.google.common.collect.ImmutableMap;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.model.ColumnFamily;
+
+
+/**
+ * Utilities for Cassandra tests
+ */
+public class TestUtils {
+
+ private static final Logger log = LoggerFactory.getLogger( TestUtils.class );
+
+ /**
+ * Create the kespace, ignore exceptions if it already exists
+ * @param keyspace
+ */
+ public static void createKeyspace(final Keyspace keyspace){
+
+ ImmutableMap.Builder<Object, Object> strategyOptions = ImmutableMap.builder().put( "replication_factor", "1" );
+
+ ImmutableMap<String, Object> options = ImmutableMap.<String, Object>builder().put( "strategy_class",
+ "org.apache.cassandra.locator.SimpleStrategy" ).put( "strategy_options", strategyOptions.build() )
+ .build();
+
+
+ try {
+ keyspace.createKeyspace( options );
+ }
+ catch ( Throwable t ) {
+ log.error( "Error on creating keyspace, ignoring", t );
+ }
+
+
+
+ }
+
+
+ public static <K, C> void createColumnFamiliy(final Keyspace keyspace, final ColumnFamily<K, C> columnFamily, final Map<String, Object> options){
+ try{
+ keyspace.createColumnFamily( columnFamily, new HashMap<String, Object>() );
+ }catch(Exception e){
+ log.error( "Error on creating column family, ignoring" , e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java
index ee89e0f..43f5a0c 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java
@@ -51,6 +51,15 @@ public class CassandraRule extends EnvironResource {
cassandraFig = injector.getInstance( CassandraFig.class );
}
+
+ /**
+ * Get the cassandra fig
+ * @return
+ */
+ public CassandraFig getCassandraFig(){
+ return cassandraFig;
+ }
+
@Override
protected void before() throws Throwable {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/OrderedMergeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/OrderedMergeTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/OrderedMergeTest.java
index 72e49f7..07e2e58 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/OrderedMergeTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/rx/OrderedMergeTest.java
@@ -107,7 +107,7 @@ public class OrderedMergeTest {
Observable<Integer> ordered =
- OrderedMerge.orderedMerge( new IntegerComparator(), 10, expected1, expected2, expected3 );
+ OrderedMerge.orderedMerge( new ReverseIntegerComparator(), 10, expected1, expected2, expected3 );
final CountDownLatch latch = new CountDownLatch( 1 );
final List<Integer> results = new ArrayList();
@@ -165,7 +165,7 @@ public class OrderedMergeTest {
//set our buffer size to 2. We should easily exceed this since every observable has more than 2 elements
Observable<Integer> ordered =
- OrderedMerge.orderedMerge( new IntegerComparator(), 2, expected1, expected2, expected3 );
+ OrderedMerge.orderedMerge( new ReverseIntegerComparator(), 2, expected1, expected2, expected3 );
final CountDownLatch latch = new CountDownLatch( 1 );
final List<Integer> results = new ArrayList();
@@ -228,7 +228,7 @@ public class OrderedMergeTest {
Observable<Integer> ordered =
- OrderedMerge.orderedMerge( new IntegerComparator(), 10, expected1, expected2, expected3 );
+ OrderedMerge.orderedMerge( new ReverseIntegerComparator(), 10, expected1, expected2, expected3 );
final CountDownLatch latch = new CountDownLatch( 1 );
final List<Integer> results = new ArrayList();
@@ -359,7 +359,7 @@ public class OrderedMergeTest {
* proceed
*/
Observable<Integer> ordered =
- OrderedMerge.orderedMerge( new IntegerComparator(), 2, expected1, expected2, expected3 );
+ OrderedMerge.orderedMerge( new ReverseIntegerComparator(), 2, expected1, expected2, expected3 );
final CountDownLatch latch = new CountDownLatch( 1 );
@@ -399,6 +399,144 @@ public class OrderedMergeTest {
}
+ /**
+ * Tests that with a buffer size much smaller than our inputs, we successfully block observables from
+ * producing values when our pressure gets too high. Eventually, one of these events should begin production, eventually
+ * draining all values
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testDuplicateOrderingCorrect() throws InterruptedException {
+
+ List<Integer> expected1List = Arrays.asList( 10, 5, 4, 3, 2, 1 );
+
+ Observable<Integer> expected1 = Observable.from( expected1List ).subscribeOn( Schedulers.io() );
+
+ List<Integer> expected2List = Arrays.asList( 9, 8, 7, 6, 5 );
+
+ Observable<Integer> expected2 = Observable.from( expected2List ).subscribeOn( Schedulers.io() );
+
+
+ List<Integer> expected3List = Arrays.asList( 9, 6, 5, 3, 2, 1, 0 );
+
+ Observable<Integer> expected3 = Observable.from( expected3List ).subscribeOn( Schedulers.io() );
+
+
+ /**
+ * Fails because our first observable will have to buffer the last 4 elements while waiting for the others to
+ * proceed
+ */
+ Observable<Integer> ordered =
+ OrderedMerge.orderedMerge( new ReverseIntegerComparator(), 2, expected1, expected2, expected3 );
+
+
+ final CountDownLatch latch = new CountDownLatch( 1 );
+ final List<Integer> results = new ArrayList();
+
+ ordered.subscribe( new Subscriber<Integer>() {
+ @Override
+ public void onCompleted() {
+ latch.countDown();
+ }
+
+
+ @Override
+ public void onError( final Throwable e ) {
+ e.printStackTrace();
+ fail( "An error was thrown " );
+ }
+
+
+ @Override
+ public void onNext( final Integer integer ) {
+ log.info( "onNext invoked with {}", integer );
+ results.add( integer );
+ }
+ } );
+
+ latch.await();
+
+ List<Integer> expected = Arrays.asList( 10, 9, 9, 8, 7, 6, 6, 5, 5, 5, 4, 3, 3, 2, 2, 1, 1, 0);
+
+ assertEquals( expected.size(), results.size() );
+
+
+ for ( int i = 0; i < expected.size(); i++ ) {
+ assertEquals( "Same element expected", expected.get( i ), results.get( i ) );
+ }
+ }
+
+
+ /**
+ * Tests that with a buffer size much smaller than our inputs, we successfully block observables from
+ * producing values when our pressure gets too high. Eventually, one of these events should begin production, eventually
+ * draining all values
+ *
+ * @throws InterruptedException
+ */
+ @Test
+ public void testDuplicateOrderingCorrectComparator() throws InterruptedException {
+
+ List<Integer> expected1List = Arrays.asList( 1, 2, 3, 4, 5, 10 );
+
+ Observable<Integer> expected1 = Observable.from( expected1List ).subscribeOn( Schedulers.io() );
+
+ List<Integer> expected2List = Arrays.asList( 5, 6, 7, 8, 9 );
+
+ Observable<Integer> expected2 = Observable.from( expected2List ).subscribeOn( Schedulers.io() );
+
+
+ List<Integer> expected3List = Arrays.asList( 0, 1, 2, 3, 5, 6, 9 );
+
+ Observable<Integer> expected3 = Observable.from( expected3List ).subscribeOn( Schedulers.io() );
+
+
+ /**
+ * Fails because our first observable will have to buffer the last 4 elements while waiting for the others to
+ * proceed
+ */
+ Observable<Integer> ordered =
+ OrderedMerge.orderedMerge( new IntegerComparator(), 2, expected1, expected2, expected3 );
+
+
+ final CountDownLatch latch = new CountDownLatch( 1 );
+ final List<Integer> results = new ArrayList();
+
+ ordered.subscribe( new Subscriber<Integer>() {
+ @Override
+ public void onCompleted() {
+ latch.countDown();
+ }
+
+
+ @Override
+ public void onError( final Throwable e ) {
+ e.printStackTrace();
+ fail( "An error was thrown " );
+ }
+
+
+ @Override
+ public void onNext( final Integer integer ) {
+ log.info( "onNext invoked with {}", integer );
+ results.add( integer );
+ }
+ } );
+
+ latch.await();
+
+ List<Integer> expected = Arrays.asList( 0, 1, 1,2, 2, 3, 3,4, 5, 5, 5, 6, 6, 7,8, 9, 9,10 );
+
+ assertEquals( expected.size(), results.size() );
+
+
+ for ( int i = 0; i < expected.size(); i++ ) {
+ assertEquals( "Same element expected", expected.get( i ), results.get( i ) );
+ }
+ }
+
+
private static class IntegerComparator implements Comparator<Integer> {
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/resources/log4j.properties b/stack/corepersistence/common/src/test/resources/log4j.properties
new file mode 100644
index 0000000..08d897c
--- /dev/null
+++ b/stack/corepersistence/common/src/test/resources/log4j.properties
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# suppress inspection "UnusedProperty" for whole file
+log4j.rootLogger=INFO,stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %p %c{3}.%M(%L)<%t>- %m%n
+
+log4j.logger.org.safehaus.chop.plugin=DEBUG
+log4j.logger.org.safehaus.guicyfig=ERROR
+log4j.logger.org.safehaus.chop.api.store.amazon=DEBUG
+log4j.logger.org.apache.http=ERROR
+log4j.logger.com.amazonaws.request=ERROR
+log4j.logger.cassandra.db=ERROR
+
+#log4j.logger.org.apache.usergrid=DEBUG
+
+log4j.logger.org.apache.usergrid.persistence.graph=TRACE
+log4j.logger.org.apache.usergrid.persistence.core.rx=TRACE
+#log4j.logger.org.apache.usergrid.persistence.graph.serialization.impl.parse=TRACE
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/common/src/test/resources/usergrid.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/resources/usergrid.properties b/stack/corepersistence/common/src/test/resources/usergrid.properties
new file mode 100644
index 0000000..febda88
--- /dev/null
+++ b/stack/corepersistence/common/src/test/resources/usergrid.properties
@@ -0,0 +1 @@
+# No properties in our test env
[3/5] Updated OrderedMerge to use a faster implementation at runtime.
After initialization,
it's an O(1) emit operation as long as our produces are fast enough.
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowTypeSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowTypeSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowTypeSerializer.java
deleted file mode 100644
index 2edea56..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowTypeSerializer.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-
-package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
-
-
-import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
-import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-
-
-public class RowTypeSerializer implements CompositeFieldSerializer<RowKeyType> {
-
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
-
- @Override
- public void toComposite( final CompositeBuilder builder, final RowKeyType keyType ) {
-
- //add the row id to the composite
- ID_SER.toComposite( builder, keyType.nodeId );
-
- builder.addString( keyType.edgeType );
- builder.addString( keyType.idType );
-
- builder.addLong( keyType.shardId );
- }
-
-
- @Override
- public RowKeyType fromComposite( final CompositeParser composite ) {
-
- final Id id = ID_SER.fromComposite( composite );
- final String edgeType = composite.readString();
- final String idType = composite.readString();
- final long shard = composite.readLong();
-
- return new RowKeyType( id, edgeType, idType, shard);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIterator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIterator.java
index c8a884b..f1b5108 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIterator.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIterator.java
@@ -6,32 +6,51 @@ import java.util.NoSuchElementException;
import org.apache.commons.collections4.iterators.PushbackIterator;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupCompaction;
import com.google.common.base.Preconditions;
+import rx.schedulers.Schedulers;
+
/**
- * Utility class that will take an iterator of all shards, and combine them into an iterator
- * of ShardEntryGroups. These groups can then be used in a distributed system to handle concurrent reads and writes
+ * Utility class that will take an iterator of all shards, and combine them into an iterator of ShardEntryGroups. These
+ * groups can then be used in a distributed system to handle concurrent reads and writes
*/
public class ShardEntryGroupIterator implements Iterator<ShardEntryGroup> {
- private ShardEntryGroup next;
+ private final ShardGroupCompaction shardGroupCompaction;
private final PushbackIterator<Shard> sourceIterator;
private final long minDelta;
+ private final ApplicationScope scope;
+ private final DirectedEdgeMeta directedEdgeMeta;
+
+
+ private ShardEntryGroup next;
/**
* Create a shard iterator
- * @param shardIterator The iterator of all shards. Order is expected to be by the shard index from Long.MAX to Long.MIN
+ *
+ * @param shardIterator The iterator of all shards. Order is expected to be by the shard index from Long.MAX to
+ * Long.MIN
* @param minDelta The minimum delta we allow to consider shards the same group
*/
- public ShardEntryGroupIterator( final Iterator<Shard> shardIterator, final long minDelta ) {
- Preconditions.checkArgument(shardIterator.hasNext(), "Shard iterator must have shards present");
+ public ShardEntryGroupIterator( final Iterator<Shard> shardIterator, final long minDelta,
+ final ShardGroupCompaction shardGroupCompaction, final ApplicationScope scope,
+ final DirectedEdgeMeta directedEdgeMeta ) {
+
+
+ Preconditions.checkArgument( shardIterator.hasNext(), "Shard iterator must have shards present" );
+ this.scope = scope;
+ this.directedEdgeMeta = directedEdgeMeta;
this.sourceIterator = new PushbackIterator( shardIterator );
+ this.shardGroupCompaction = shardGroupCompaction;
this.minDelta = minDelta;
}
@@ -78,7 +97,7 @@ public class ShardEntryGroupIterator implements Iterator<ShardEntryGroup> {
*/
while ( sourceIterator.hasNext() ) {
- if(next == null){
+ if ( next == null ) {
next = new ShardEntryGroup( minDelta );
}
@@ -92,9 +111,13 @@ public class ShardEntryGroupIterator implements Iterator<ShardEntryGroup> {
sourceIterator.pushback( shard );
+
break;
}
-
+ //now perform the audit (maybe)
+ if(next != null) {
+ shardGroupCompaction.evaluateShardGroup( scope, directedEdgeMeta, next );
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
index c566d43..5076424 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardGroupCompactionImpl.java
@@ -23,21 +23,32 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
import java.nio.charset.Charset;
-import java.util.BitSet;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.RejectedExecutionHandler;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.annotation.Nullable;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.usergrid.persistence.core.consistency.TimeService;
-import org.apache.usergrid.persistence.core.rx.ObservableIterator;
+import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization;
@@ -47,20 +58,22 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntry
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupCompaction;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization;
import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import com.google.common.base.Preconditions;
-import com.google.common.hash.HashCode;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
+import com.google.common.hash.PrimitiveSink;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import com.google.inject.Singleton;
-
-import rx.Observable;
-import rx.functions.Action0;
-import rx.functions.Action1;
-import rx.functions.Func1;
-import rx.schedulers.Schedulers;
+import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.MutationBatch;
/**
@@ -70,27 +83,33 @@ import rx.schedulers.Schedulers;
public class ShardGroupCompactionImpl implements ShardGroupCompaction {
+ private static final Logger LOG = LoggerFactory.getLogger( ShardGroupCompactionImpl.class );
+
+
private static final Charset CHARSET = Charset.forName( "UTF-8" );
private static final HashFunction MURMUR_128 = Hashing.murmur3_128();
+
+ private final ListeningExecutorService executorService;
private final TimeService timeService;
private final GraphFig graphFig;
private final NodeShardAllocation nodeShardAllocation;
private final ShardedEdgeSerialization shardedEdgeSerialization;
private final EdgeColumnFamilies edgeColumnFamilies;
+ private final Keyspace keyspace;
private final EdgeShardSerialization edgeShardSerialization;
-
private final Random random;
private final ShardCompactionTaskTracker shardCompactionTaskTracker;
+ private final ShardAuditTaskTracker shardAuditTaskTracker;
@Inject
public ShardGroupCompactionImpl( final TimeService timeService, final GraphFig graphFig,
final NodeShardAllocation nodeShardAllocation,
final ShardedEdgeSerialization shardedEdgeSerialization,
- final EdgeColumnFamilies edgeColumnFamilies,
+ final EdgeColumnFamilies edgeColumnFamilies, final Keyspace keyspace,
final EdgeShardSerialization edgeShardSerialization ) {
this.timeService = timeService;
@@ -98,157 +117,343 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
this.nodeShardAllocation = nodeShardAllocation;
this.shardedEdgeSerialization = shardedEdgeSerialization;
this.edgeColumnFamilies = edgeColumnFamilies;
+ this.keyspace = keyspace;
this.edgeShardSerialization = edgeShardSerialization;
this.random = new Random();
this.shardCompactionTaskTracker = new ShardCompactionTaskTracker();
+ this.shardAuditTaskTracker = new ShardAuditTaskTracker();
+
+ executorService = MoreExecutors.listeningDecorator(
+ new MaxSizeThreadPool( graphFig.getShardAuditWorkerCount(), graphFig.getShardAuditWorkerQueueSize() ) );
}
- @Override
- public Set<Shard> compact( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta,
- final ShardEntryGroup group ) {
+ /**
+ * Execute the compaction task. Will return the status the operations performed
+ *
+ * @param group The shard entry group to compact
+ *
+ * @return The result of the compaction operation
+ */
+ public CompactionResult compact( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta,
+ final ShardEntryGroup group ) {
+
+
final long startTime = timeService.getCurrentTime();
+
Preconditions.checkNotNull( group, "group cannot be null" );
Preconditions.checkArgument( group.isCompactionPending(), "Compaction is pending" );
- Preconditions.checkArgument( group.shouldCompact( startTime ), "Compaction can now be run" );
+ Preconditions.checkArgument( group.shouldCompact( startTime ),
+ "Compaction cannot be run yet. Ignoring compaction." );
- /**
- * It's already compacting, don't do anything
- */
- if (!shardCompactionTaskTracker.shouldStartCompaction( scope, edgeMeta, group )){
- return Collections.emptySet();
- }
+ final CompactionResult.CompactionBuilder resultBuilder = CompactionResult.builder();
final Shard targetShard = group.getCompactionTarget();
- final Collection<Shard> sourceShards = group.getReadShards();
+ final Set<Shard> sourceShards = new HashSet<>( group.getReadShards() );
+ //remove the target
+ sourceShards.remove( targetShard );
- Observable.create( new ObservableIterator<MarkedEdge>( "Shard_Repair" ) {
- @Override
- protected Iterator<MarkedEdge> getIterator() {
- return edgeMeta.loadEdges( shardedEdgeSerialization, edgeColumnFamilies, scope, group.getReadShards(), Long.MAX_VALUE );
- }
- } ).buffer( graphFig.getScanPageSize() ).doOnNext( new Action1<List<MarkedEdge>>() {
- @Override
- public void call( final List<MarkedEdge> markedEdges ) {
+ final UUID timestamp = UUIDGenerator.newTimeUUID();
- }
- }).doOnNext( new Action1<List<MarkedEdge>>() {
- @Override
- public void call( final List<MarkedEdge> markedEdges ) {
+ final long newShardPivot = targetShard.getShardIndex();
- }
- } );
+ final int maxWorkSize = graphFig.getScanPageSize();
+ final MutationBatch newRowBatch = keyspace.prepareMutationBatch();
+ final MutationBatch deleteRowBatch = keyspace.prepareMutationBatch();
+ /**
+ * As we move edges, we want to keep track of it
+ */
+ long edgeCount = 0;
- return null;
- }
+ for ( Shard sourceShard : sourceShards ) {
+ Iterator<MarkedEdge> edges = edgeMeta.loadEdges( shardedEdgeSerialization, edgeColumnFamilies, scope,
+ Collections.singleton( sourceShard ), Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING );
+ while ( edges.hasNext() ) {
+ final MarkedEdge edge = edges.next();
- @Override
- public AuditResult evaluateShardGroup( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta,
- final ShardEntryGroup group ) {
+ final long edgeTimestamp = edge.getTimestamp();
+ /**
+ * The edge is within a different shard, break
+ */
+ if ( edgeTimestamp < newShardPivot ) {
+ break;
+ }
- final double repairChance = random.nextDouble();
- //don't repair
- if ( repairChance > graphFig.getShardRepairChance() ) {
- return AuditResult.NOT_CHECKED;
+ newRowBatch.mergeShallow(
+ edgeMeta.writeEdge( shardedEdgeSerialization, edgeColumnFamilies, scope, targetShard, edge,
+ timestamp ) );
+
+ deleteRowBatch.mergeShallow(
+ edgeMeta.deleteEdge( shardedEdgeSerialization, edgeColumnFamilies, scope, sourceShard, edge,
+ timestamp ) );
+
+ edgeCount++;
+
+ //if we're at our count, execute the mutation of writing the edges to the new row, then remove them
+ //from the old rows
+ if ( edgeCount % maxWorkSize == 0 ) {
+
+ try {
+ HystrixCassandra.async( newRowBatch );
+ HystrixCassandra.async( deleteRowBatch );
+ }
+ catch ( Throwable t ) {
+ LOG.error( "Unable to move edges from shard {} to shard {}", sourceShard, targetShard );
+ }
+ }
+ }
}
+ try {
+ HystrixCassandra.async( newRowBatch );
+ HystrixCassandra.async( deleteRowBatch );
+ }
+ catch ( Throwable t ) {
+ LOG.error( "Unable to move edges to target shard {}", targetShard );
+ }
+
+
+ LOG.info( "Finished compacting {} shards and moved {} edges", sourceShards, edgeCount );
+
+ resultBuilder.withCopiedEdges( edgeCount ).withSourceShards( sourceShards ).withTargetShard( targetShard );
+
/**
- * We don't have a compaction pending. Run an audit on the shards
+ * We didn't move anything this pass, mark the shard as compacted. If we move something, it means that we missed it on the first pass
+ * or someone is still not writing to the target shard only.
*/
- if ( !group.isCompactionPending() ) {
+ if ( edgeCount == 0 ) {
- /**
- * Check if we should allocate, we may want to
- */
+ //now that we've marked our target as compacted, we can successfully remove any shards that are not
+ // compacted themselves in the sources
+
+ final MutationBatch shardRemovalRollup = keyspace.prepareMutationBatch();
+
+ for ( Shard source : sourceShards ) {
+
+ //if we can't safely delete it, don't do so
+ if ( !group.canBeDeleted( source ) ) {
+ continue;
+ }
- final boolean created = nodeShardAllocation.auditShard( scope, group, edgeMeta );
+ LOG.info( "Source shards have been fully drained. Removing shard {}", source );
+ final MutationBatch shardRemoval = edgeShardSerialization.removeShardMeta( scope, source, edgeMeta );
+ shardRemovalRollup.mergeShallow( shardRemoval );
- if ( !created ) {
- return AuditResult.CHECKED_NO_OP;
+ resultBuilder.withRemovedShard( source );
}
- return AuditResult.CHECKED_CREATED;
+ HystrixCassandra.async( shardRemovalRollup );
+
+
+ LOG.info( "Shard has been fully compacted. Marking shard {} as compacted in Cassandra", targetShard );
+
+ //Overwrite our shard index with a newly created one that has been marked as compacted
+ Shard compactedShard = new Shard( targetShard.getShardIndex(), timeService.getCurrentTime(), true );
+ final MutationBatch updateMark = edgeShardSerialization.writeShardMeta( scope, compactedShard, edgeMeta );
+ HystrixCassandra.async( updateMark );
+
+ resultBuilder.withCompactedShard( compactedShard );
}
- //check our taskmanager
+ return resultBuilder.build();
+ }
+ @Override
+ public ListenableFuture<AuditResult> evaluateShardGroup( final ApplicationScope scope,
+ final DirectedEdgeMeta edgeMeta,
+ final ShardEntryGroup group ) {
+
+ final double repairChance = random.nextDouble();
+
+
+ //don't audit, we didn't hit our chance
+ if ( repairChance > graphFig.getShardRepairChance() ) {
+ return Futures.immediateFuture( AuditResult.NOT_CHECKED );
+ }
+
/**
- * Do the compaction
+ * Try and submit. During back pressure, we may not be able to submit, that's ok. Better to drop than to
+ * hose the system
*/
- if ( group.shouldCompact( timeService.getCurrentTime() ) ) {
- compact( scope, edgeMeta, group );
- return AuditResult.COMPACTING;
- }
+ ListenableFuture<AuditResult> future = executorService.submit( new Callable<AuditResult>() {
+ @Override
+ public AuditResult call() throws Exception {
+
+
+ /**
+ * We don't have a compaction pending. Run an audit on the shards
+ */
+ if ( !group.isCompactionPending() ) {
+
+ /**
+ * Check if we should allocate, we may want to
+ */
+
+ /**
+ * It's already compacting, don't do anything
+ */
+ if ( !shardAuditTaskTracker.canStartTask( scope, edgeMeta, group ) ) {
+ return AuditResult.CHECKED_NO_OP;
+ }
+
+ try {
+
+ final boolean created = nodeShardAllocation.auditShard( scope, group, edgeMeta );
+ if ( !created ) {
+ return AuditResult.CHECKED_NO_OP;
+ }
+ }
+ finally {
+ shardAuditTaskTracker.complete( scope, edgeMeta, group );
+ }
+
+
+ return AuditResult.CHECKED_CREATED;
+ }
+
+ //check our taskmanager
+
+
+ /**
+ * Do the compaction
+ */
+ if ( group.shouldCompact( timeService.getCurrentTime() ) ) {
+ /**
+ * It's already compacting, don't do anything
+ */
+ if ( !shardCompactionTaskTracker.canStartTask( scope, edgeMeta, group ) ) {
+ return AuditResult.COMPACTING;
+ }
+
+ /**
+ * We use a finally b/c we always want to remove the task track
+ */
+ try {
+ CompactionResult result = compact( scope, edgeMeta, group );
+ LOG.info( "Compaction result for compaction of scope {} with edge meta data of {} and shard group {} is {}", new Object[]{scope, edgeMeta, group, result} );
+ }
+ finally {
+ shardCompactionTaskTracker.complete( scope, edgeMeta, group );
+ }
+ return AuditResult.COMPACTED;
+ }
+
+ //no op, there's nothing we need to do to this shard
+ return AuditResult.NOT_CHECKED;
+ }
+ } );
+
+ /**
+ * Log our success or failures for debugging purposes
+ */
+ Futures.addCallback( future, new FutureCallback<AuditResult>() {
+ @Override
+ public void onSuccess( @Nullable final AuditResult result ) {
+ LOG.debug( "Successfully completed audit of task {}", result );
+ }
- //no op, there's nothing we need to do to this shard
- return AuditResult.NOT_CHECKED;
+
+ @Override
+ public void onFailure( final Throwable t ) {
+ LOG.error( "Unable to perform audit. Exception is ", t );
+ }
+ } );
+
+ return future;
}
- private static final class ShardCompactionTaskTracker {
- private BitSet runningTasks = new BitSet();
+ /**
+ * Inner class used to track running tasks per instance
+ */
+ private static abstract class TaskTracker {
+
+ private static final Boolean TRUE = true;
+
+ private ConcurrentHashMap<Long, Boolean> runningTasks = new ConcurrentHashMap<>();
/**
* Sets this data into our scope to signal it's running to stop other threads from attempting to run
- * @param scope
- * @param edgeMeta
- * @param group
- * @return
*/
- public boolean shouldStartCompaction( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta,
+ public boolean canStartTask( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta,
ShardEntryGroup group ) {
- final int hash = doHash( scope, edgeMeta, group ).asInt();
+ final Long hash = doHash( scope, edgeMeta, group ).hash().asLong();
- if(runningTasks.get( hash )){
- return false;
- }
-
- runningTasks.set( hash );
+ final Boolean returned = runningTasks.putIfAbsent( hash, TRUE );
- return true;
+ /**
+ * Someone already put the value
+ */
+ return returned == null;
}
/**
* Mark this entry group as complete
- * @param scope
- * @param edgeMeta
- * @param group
*/
- public void complete( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta,
- ShardEntryGroup group ) {
- final int hash = doHash( scope, edgeMeta, group ).asInt();
- runningTasks.clear( hash );
+ public void complete( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta, ShardEntryGroup group ) {
+ final long hash = doHash( scope, edgeMeta, group ).hash().asLong();
+ runningTasks.remove( hash );
}
+ protected abstract Hasher doHash( final ApplicationScope scope, final DirectedEdgeMeta directedEdgeMeta,
+ final ShardEntryGroup shardEntryGroup );
+ }
+
+
+ /**
+ * Task tracker for shard compaction
+ */
+ private static final class ShardCompactionTaskTracker extends ShardAuditTaskTracker {
+
/**
* Hash our data into a consistent long
- * @param scope
- * @param directedEdgeMeta
- * @param shardEntryGroup
- * @return
*/
- private HashCode doHash( final ApplicationScope scope, final DirectedEdgeMeta directedEdgeMeta,
+ protected Hasher doHash( final ApplicationScope scope, final DirectedEdgeMeta directedEdgeMeta,
+ final ShardEntryGroup shardEntryGroup ) {
+
+ final Hasher hasher = super.doHash( scope, directedEdgeMeta, shardEntryGroup );
+
+ //add our compaction target to the hash
+ final Shard compactionTarget = shardEntryGroup.getCompactionTarget();
+
+ hasher.putLong( compactionTarget.getShardIndex() );
+
+
+ return hasher;
+ }
+ }
+
+
+ /**
+ * Task tracker for shard audit
+ */
+ private static class ShardAuditTaskTracker extends TaskTracker {
+
+ /**
+ * Hash our data into a consistent long
+ */
+ protected Hasher doHash( final ApplicationScope scope, final DirectedEdgeMeta directedEdgeMeta,
final ShardEntryGroup shardEntryGroup ) {
final Hasher hasher = MURMUR_128.newHasher();
@@ -273,35 +478,141 @@ public class ShardGroupCompactionImpl implements ShardGroupCompaction {
}
//add our compaction target to the hash
- final Shard compactionTarget = shardEntryGroup.getCompactionTarget();
- hasher.putLong( compactionTarget.getShardIndex() );
-
- return hasher.hash();
+ return hasher;
}
- private void addToHash( final Hasher hasher, final Id id ) {
+ protected void addToHash( final PrimitiveSink into, final Id id ) {
final UUID nodeUuid = id.getUuid();
final String nodeType = id.getType();
- hasher.putLong( nodeUuid.getMostSignificantBits() ).putLong( nodeUuid.getLeastSignificantBits() )
- .putString( nodeType, CHARSET );
+ into.putLong( nodeUuid.getMostSignificantBits() ).putLong( nodeUuid.getLeastSignificantBits() )
+ .putString( nodeType, CHARSET );
+ }
+ }
+
+
+ /**
+ * Create a thread pool that will reject work if our audit tasks become overwhelmed
+ */
+ private final class MaxSizeThreadPool extends ThreadPoolExecutor {
+
+ public MaxSizeThreadPool( final int workerSize, final int queueLength ) {
+ super( 1, workerSize, 30, TimeUnit.SECONDS, new ArrayBlockingQueue<Runnable>( queueLength ),
+ new CompactionThreadFactory(), new RejectionLogger() );
+ }
+ }
+
+
+ private final class CompactionThreadFactory implements ThreadFactory {
+
+ private final AtomicLong threadCounter = new AtomicLong();
+
+
+ @Override
+ public Thread newThread( final Runnable r ) {
+ final long newValue = threadCounter.incrementAndGet();
+
+ return new Thread( r, "Graph-Shard-Compaction-" + newValue );
}
}
- private enum StartResult{
- /**
- * Returned if the compaction was started
- */
- STARTED,
+ private final class RejectionLogger implements RejectedExecutionHandler {
+
+
+ @Override
+ public void rejectedExecution( final Runnable r, final ThreadPoolExecutor executor ) {
+ LOG.warn( "Audit queue full, rejecting audit task {}", r );
+ }
+ }
+
+
+
+ public static final class CompactionResult {
+
+ public final long copiedEdges;
+ public final Shard targetShard;
+ public final Set<Shard> sourceShards;
+ public final Set<Shard> removedShards;
+ public final Shard compactedShard;
+
+
+
+ private CompactionResult( final long copiedEdges, final Shard targetShard, final Set<Shard> sourceShards,
+ final Set<Shard> removedShards, final Shard compactedShard ) {
+ this.copiedEdges = copiedEdges;
+ this.targetShard = targetShard;
+ this.compactedShard = compactedShard;
+ this.sourceShards = Collections.unmodifiableSet( sourceShards );
+ this.removedShards = Collections.unmodifiableSet( removedShards );
+ }
+
/**
- * Returned if we are running the compaction
+ * Create a builder to use to create the result
*/
- RUNNING;
+ public static CompactionBuilder builder() {
+ return new CompactionBuilder();
+ }
+
+
+ @Override
+ public String toString() {
+ return "CompactionResult{" +
+ "copiedEdges=" + copiedEdges +
+ ", targetShard=" + targetShard +
+ ", sourceShards=" + sourceShards +
+ ", removedShards=" + removedShards +
+ ", compactedShard=" + compactedShard +
+ '}';
+ }
+
+
+ public static final class CompactionBuilder {
+ private long copiedEdges;
+ private Shard targetShard;
+ private Set<Shard> sourceShards;
+ private Set<Shard> removedShards = new HashSet<>();
+ private Shard compactedShard;
+
+
+ public CompactionBuilder withCopiedEdges( final long copiedEdges ) {
+ this.copiedEdges = copiedEdges;
+ return this;
+ }
+
+
+ public CompactionBuilder withTargetShard( final Shard targetShard ) {
+ this.targetShard = targetShard;
+ return this;
+ }
+
+
+ public CompactionBuilder withSourceShards( final Set<Shard> sourceShards ) {
+ this.sourceShards = sourceShards;
+ return this;
+ }
+
+
+ public CompactionBuilder withRemovedShard( final Shard removedShard ) {
+ this.removedShards.add( removedShard );
+ return this;
+ }
+
+
+ public CompactionBuilder withCompactedShard( final Shard compactedShard ) {
+ this.compactedShard = compactedShard;
+ return this;
+ }
+
+
+ public CompactionResult build() {
+ return new CompactionResult( copiedEdges, targetShard, sourceShards, removedShards, compactedShard );
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
index 030e4a7..b0523e1 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardedEdgeSerializationImpl.java
@@ -22,9 +22,11 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
import java.util.Collection;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.UUID;
+import javax.annotation.Nullable;
import javax.inject.Inject;
import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
@@ -49,9 +51,16 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators.DescendingTimestampComparator;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators.OrderedComparator;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators
+ .SourceDirectedEdgeDescendingComparator;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators
+ .TargetDirectedEdgeDescendingComparator;
import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.google.common.base.Function;
import com.google.common.base.Optional;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
@@ -256,6 +265,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
final Shard shard, final boolean isDeleted ) {
batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).deleteColumn( edge );
+ writeEdgeShardStrategy.increment( scope, shard, -1, directedEdgeMeta );
}
}.createBatch( scope, shards, timestamp );
}
@@ -278,6 +288,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
batch.withRow( columnFamilies.getSourceNodeTargetTypeCfName(), ScopedRowKey.fromKey( scope, rowKey ) )
.deleteColumn( edge );
+ writeEdgeShardStrategy.increment( scope, shard, -1, directedEdgeMeta );
}
}.createBatch( scope, shards, timestamp );
}
@@ -297,6 +308,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
final Shard shard, final boolean isDeleted ) {
batch.withRow( columnFamily, ScopedRowKey.fromKey( scope, rowKey ) ).deleteColumn( edge );
+ writeEdgeShardStrategy.increment( scope, shard, -1, directedEdgeMeta );
}
}.createBatch( scope, shards, timestamp );
}
@@ -319,6 +331,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
batch.withRow( columnFamilies.getTargetNodeSourceTypeCfName(), ScopedRowKey.fromKey( scope, rowKey ) )
.deleteColumn( edge );
+ writeEdgeShardStrategy.increment( scope, shard, -1, directedEdgeMeta );
}
}.createBatch( scope, shards, timestamp );
}
@@ -338,6 +351,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
final boolean isDeleted ) {
batch.withRow( columnFamilies.getGraphEdgeVersions(), ScopedRowKey.fromKey( scope, rowKey ) )
.deleteColumn( column );
+ writeEdgeShardStrategy.increment( scope, shard, -1, directedEdgeMeta );
}
}.createBatch( scope, shards, timestamp );
}
@@ -357,8 +371,16 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
columnFamilies.getGraphEdgeVersions();
final Serializer<Long> serializer = columnFamily.getColumnSerializer();
+
+ final OrderedComparator<MarkedEdge> comparator = new OrderedComparator<>( DescendingTimestampComparator.INSTANCE, search.getOrder());
+
+
+
+
final EdgeSearcher<EdgeRowKey, Long, MarkedEdge> searcher =
- new EdgeSearcher<EdgeRowKey, Long, MarkedEdge>( scope, maxTimestamp, search.last(), shards ) {
+ new EdgeSearcher<EdgeRowKey, Long, MarkedEdge>( scope, shards, search.getOrder(), comparator, maxTimestamp,
+ search.last().transform( TRANSFORM ) ) {
+
@Override
protected Serializer<Long> getSerializer() {
@@ -367,11 +389,11 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
- public void setRange( final RangeBuilder builder ) {
+ public void buildRange( final RangeBuilder builder ) {
if ( last.isPresent() ) {
- super.setRange( builder );
+ super.buildRange( builder );
return;
}
@@ -387,7 +409,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
- protected Long getStartColumn( final Edge last ) {
+ protected Long createColumn( final MarkedEdge last ) {
return last.getTimestamp();
}
@@ -398,10 +420,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
}
- @Override
- public int compare( final MarkedEdge o1, final MarkedEdge o2 ) {
- return Long.compare( o1.getTimestamp(), o2.getTimestamp() );
- }
+
};
return new ShardsColumnIterator<>( searcher, columnFamily, keyspace, cassandraConfig.getReadCL(),
@@ -411,23 +430,27 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
public Iterator<MarkedEdge> getEdgesFromSource( final EdgeColumnFamilies columnFamilies,
- final ApplicationScope scope, final SearchByEdgeType edgeType,
+ final ApplicationScope scope, final SearchByEdgeType search,
final Collection<Shard> shards ) {
ValidationUtils.validateApplicationScope( scope );
- GraphValidation.validateSearchByEdgeType( edgeType );
+ GraphValidation.validateSearchByEdgeType( search );
- final Id sourceId = edgeType.getNode();
- final String type = edgeType.getType();
- final long maxTimestamp = edgeType.getMaxTimestamp();
+ final Id sourceId = search.getNode();
+ final String type = search.getType();
+ final long maxTimestamp = search.getMaxTimestamp();
final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily =
columnFamilies.getSourceNodeCfName();
final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
- final SourceEdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
- new SourceEdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(),
- shards ) {
+ final OrderedComparator<MarkedEdge> comparator = new OrderedComparator<>( TargetDirectedEdgeDescendingComparator.INSTANCE, search.getOrder());
+
+
+
+ final EdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
+ new EdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, shards, search.getOrder(), comparator, maxTimestamp,
+ search.last().transform( TRANSFORM ) ) {
@Override
@@ -443,7 +466,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
- protected DirectedEdge getStartColumn( final Edge last ) {
+ protected DirectedEdge createColumn( final MarkedEdge last ) {
return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
}
@@ -463,24 +486,26 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
public Iterator<MarkedEdge> getEdgesFromSourceByTargetType( final EdgeColumnFamilies columnFamilies,
final ApplicationScope scope,
- final SearchByIdType edgeType,
+ final SearchByIdType search,
final Collection<Shard> shards ) {
ValidationUtils.validateApplicationScope( scope );
- GraphValidation.validateSearchByEdgeType( edgeType );
+ GraphValidation.validateSearchByEdgeType( search );
- final Id targetId = edgeType.getNode();
- final String type = edgeType.getType();
- final String targetType = edgeType.getIdType();
- final long maxTimestamp = edgeType.getMaxTimestamp();
+ final Id targetId = search.getNode();
+ final String type = search.getType();
+ final String targetType = search.getIdType();
+ final long maxTimestamp = search.getMaxTimestamp();
final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> columnFamily =
columnFamilies.getSourceNodeTargetTypeCfName();
final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+ final OrderedComparator<MarkedEdge> comparator = new OrderedComparator<>( TargetDirectedEdgeDescendingComparator.INSTANCE, search.getOrder());
+
- final SourceEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
- new SourceEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(),
- shards ) {
+ final EdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
+ new EdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, shards, search.getOrder(), comparator, maxTimestamp,
+ search.last().transform( TRANSFORM ) ) {
@Override
protected Serializer<DirectedEdge> getSerializer() {
@@ -495,7 +520,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
- protected DirectedEdge getStartColumn( final Edge last ) {
+ protected DirectedEdge createColumn( final MarkedEdge last ) {
return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
}
@@ -513,20 +538,22 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
public Iterator<MarkedEdge> getEdgesToTarget( final EdgeColumnFamilies columnFamilies, final ApplicationScope scope,
- final SearchByEdgeType edgeType, final Collection<Shard> shards ) {
+ final SearchByEdgeType search, final Collection<Shard> shards ) {
ValidationUtils.validateApplicationScope( scope );
- GraphValidation.validateSearchByEdgeType( edgeType );
+ GraphValidation.validateSearchByEdgeType( search );
- final Id targetId = edgeType.getNode();
- final String type = edgeType.getType();
- final long maxTimestamp = edgeType.getMaxTimestamp();
+ final Id targetId = search.getNode();
+ final String type = search.getType();
+ final long maxTimestamp = search.getMaxTimestamp();
final MultiTennantColumnFamily<ApplicationScope, RowKey, DirectedEdge> columnFamily =
columnFamilies.getTargetNodeCfName();
final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
- final TargetEdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
- new TargetEdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(),
- shards ) {
+ final OrderedComparator<MarkedEdge> comparator = new OrderedComparator<>( SourceDirectedEdgeDescendingComparator.INSTANCE, search.getOrder());
+
+ final EdgeSearcher<RowKey, DirectedEdge, MarkedEdge> searcher =
+ new EdgeSearcher<RowKey, DirectedEdge, MarkedEdge>( scope, shards, search.getOrder(),comparator, maxTimestamp,
+ search.last().transform( TRANSFORM ) ) {
@Override
protected Serializer<DirectedEdge> getSerializer() {
@@ -541,7 +568,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
- protected DirectedEdge getStartColumn( final Edge last ) {
+ protected DirectedEdge createColumn( final MarkedEdge last ) {
return new DirectedEdge( last.getSourceNode(), last.getTimestamp() );
}
@@ -561,24 +588,26 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
public Iterator<MarkedEdge> getEdgesToTargetBySourceType( final EdgeColumnFamilies columnFamilies,
final ApplicationScope scope,
- final SearchByIdType edgeType,
+ final SearchByIdType search,
final Collection<Shard> shards ) {
ValidationUtils.validateApplicationScope( scope );
- GraphValidation.validateSearchByEdgeType( edgeType );
+ GraphValidation.validateSearchByEdgeType( search );
- final Id targetId = edgeType.getNode();
- final String sourceType = edgeType.getIdType();
- final String type = edgeType.getType();
- final long maxTimestamp = edgeType.getMaxTimestamp();
+ final Id targetId = search.getNode();
+ final String sourceType = search.getIdType();
+ final String type = search.getType();
+ final long maxTimestamp = search.getMaxTimestamp();
final MultiTennantColumnFamily<ApplicationScope, RowKeyType, DirectedEdge> columnFamily =
columnFamilies.getTargetNodeSourceTypeCfName();
final Serializer<DirectedEdge> serializer = columnFamily.getColumnSerializer();
+ final OrderedComparator<MarkedEdge> comparator = new OrderedComparator<>( SourceDirectedEdgeDescendingComparator.INSTANCE, search.getOrder());
+
- final TargetEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
- new TargetEdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, maxTimestamp, edgeType.last(),
- shards ) {
+ final EdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge> searcher =
+ new EdgeSearcher<RowKeyType, DirectedEdge, MarkedEdge>( scope, shards, search.getOrder(), comparator, maxTimestamp,
+ search.last().transform( TRANSFORM ) ) {
@Override
protected Serializer<DirectedEdge> getSerializer() {
return serializer;
@@ -592,7 +621,7 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
@Override
- protected DirectedEdge getStartColumn( final Edge last ) {
+ protected DirectedEdge createColumn( final MarkedEdge last ) {
return new DirectedEdge( last.getTargetNode(), last.getTimestamp() );
}
@@ -608,51 +637,8 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
}
- /**
- * Class for performing searched on rows based on source id
- */
- private static abstract class SourceEdgeSearcher<R, C, T extends Edge> extends EdgeSearcher<R, C, T> {
-
- protected SourceEdgeSearcher( final ApplicationScope scope, final long maxTimestamp, final Optional<Edge> last,
- final Collection<Shard> shards ) {
- super( scope, maxTimestamp, last, shards );
- }
- public int compare( final T o1, final T o2 ) {
- int compare = Long.compare( o1.getTimestamp(), o2.getTimestamp() );
-
- if ( compare == 0 ) {
- compare = o1.getTargetNode().compareTo( o2.getTargetNode() );
- }
-
- return compare;
- }
- }
-
-
- /**
- * Class for performing searched on rows based on target id
- */
- private static abstract class TargetEdgeSearcher<R, C, T extends Edge> extends EdgeSearcher<R, C, T> {
-
- protected TargetEdgeSearcher( final ApplicationScope scope, final long maxTimestamp, final Optional<Edge> last,
- final Collection<Shard> shards ) {
- super( scope, maxTimestamp, last, shards );
- }
-
-
- public int compare( final T o1, final T o2 ) {
- int compare = Long.compare( o1.getTimestamp(), o2.getTimestamp() );
-
- if ( compare == 0 ) {
- compare = o1.getTargetNode().compareTo( o2.getTargetNode() );
- }
-
- return compare;
- }
- }
-
/**
* Simple callback to perform puts and deletes with a common row setup code
@@ -994,4 +980,27 @@ public class ShardedEdgeSerializationImpl implements ShardedEdgeSerialization {
return isDeleted;
}
}
+
+
+
+
+
+
+ private static final Function<Edge, MarkedEdge> TRANSFORM = new Function<Edge, MarkedEdge>() {
+ @Nullable
+ @Override
+ public MarkedEdge apply( @Nullable final Edge input ) {
+
+ if ( input == null ) {
+ return null;
+ }
+
+ if ( input instanceof MarkedEdge ) {
+ return ( MarkedEdge ) input;
+ }
+
+ return new SimpleMarkedEdge( input.getSourceNode(), input.getType(), input.getTargetNode(),
+ input.getTimestamp(), false );
+ }
+ };
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardsColumnIterator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardsColumnIterator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardsColumnIterator.java
index 0c7e5b5..d99d98b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardsColumnIterator.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardsColumnIterator.java
@@ -8,6 +8,7 @@ import java.util.NoSuchElementException;
import org.apache.usergrid.persistence.core.astyanax.ColumnNameIterator;
import org.apache.usergrid.persistence.core.astyanax.MultiKeyColumnNameIterator;
+import org.apache.usergrid.persistence.core.astyanax.MultiRowColumnIterator;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -86,6 +87,7 @@ public class ShardsColumnIterator<R, C, T> implements Iterator<T> {
*/
private void startIterator() {
+
/**
* If the edge is present, we need to being seeking from this
*/
@@ -94,7 +96,9 @@ public class ShardsColumnIterator<R, C, T> implements Iterator<T> {
//set the range into the search
- searcher.setRange( rangeBuilder );
+ searcher.buildRange( rangeBuilder );
+
+
/**
* Get our list of slices
@@ -102,27 +106,22 @@ public class ShardsColumnIterator<R, C, T> implements Iterator<T> {
final List<ScopedRowKey<ApplicationScope, R>> rowKeys = searcher.getRowKeys();
- final List<ColumnNameIterator<C, T>> columnNameIterators = new ArrayList<>( rowKeys.size() );
-
- for(ScopedRowKey<ApplicationScope, R> rowKey: rowKeys){
-
+ if(rowKeys.size() == 1){
+ final RowQuery<ScopedRowKey<ApplicationScope, R>, C> query =
+ keyspace.prepareQuery( cf ).setConsistencyLevel( consistencyLevel ).getKey( rowKeys.get( 0 ) )
+ .autoPaginate( true ).withColumnRange( rangeBuilder.build() );
- final RowQuery<ScopedRowKey<ApplicationScope, R>, C> query =
- keyspace.prepareQuery( cf ).setConsistencyLevel( consistencyLevel ).getKey( rowKey )
- .autoPaginate( true ).withColumnRange( rangeBuilder.build() );
-
-
- final ColumnNameIterator<C, T> columnNameIterator = new ColumnNameIterator<>( query, searcher, searcher.hasPage() );
+ currentColumnIterator = new ColumnNameIterator<>( query, searcher, searcher.hasPage() );
+ }
- columnNameIterators.add( columnNameIterator );
+ else{
+ currentColumnIterator = new MultiRowColumnIterator<>( keyspace, cf, consistencyLevel, searcher, searcher, searcher.getComparator(), rowKeys, pageSize);
}
- currentColumnIterator = new MultiKeyColumnNameIterator<>(columnNameIterators, searcher, pageSize);
-
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
index 9050b0a..ddd514b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
@@ -37,6 +37,10 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumn
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.EdgeRowKeySerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.EdgeSerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.RowSerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.RowTypeSerializer;
import com.netflix.astyanax.serializers.LongSerializer;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DescendingTimestampComparator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DescendingTimestampComparator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DescendingTimestampComparator.java
new file mode 100644
index 0000000..6e52dd5
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DescendingTimestampComparator.java
@@ -0,0 +1,43 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators;
+
+
+import java.util.Comparator;
+
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+
+
+/**
+ * Sorts longs from high to low. High is "less than" the low values;
+ *
+ */
+public class DescendingTimestampComparator implements Comparator<MarkedEdge> {
+
+ public static final DescendingTimestampComparator INSTANCE = new DescendingTimestampComparator();
+
+
+ @Override
+ public int compare( final MarkedEdge o1, final MarkedEdge o2 ) {
+ return Long.compare( o1.getTimestamp(), o2.getTimestamp() )*-1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DirectedEdgeDescendingComparator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DirectedEdgeDescendingComparator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DirectedEdgeDescendingComparator.java
new file mode 100644
index 0000000..6ff65cb
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/DirectedEdgeDescendingComparator.java
@@ -0,0 +1,69 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators;
+
+
+import java.util.Comparator;
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.fasterxml.uuid.UUIDComparator;
+import com.fasterxml.uuid.impl.UUIDUtil;
+
+
+/**
+ * Comparator for comparing edges in descending order. The first comparison is the timestamp,
+ * highest value should be first, so is considered "less". If those are equal, the UUIId is compared.
+ * this will return the UUID to compare. It will first be descending UUID, then ascending name
+ *
+ */
+public abstract class DirectedEdgeDescendingComparator implements Comparator<MarkedEdge> {
+
+ @Override
+ public int compare( final MarkedEdge first, final MarkedEdge second ) {
+
+ int compare = Long.compare( first.getTimestamp(), second.getTimestamp() ) * -1;
+
+ if(compare == 0){
+ final Id firstId = getId( first );
+ final Id secondId = getId( second );
+
+ compare = UUIDComparator.staticCompare( firstId.getUuid(), secondId.getUuid() ) * -1;
+
+ if(compare == 0){
+ compare = firstId.getType().compareTo( secondId.getType() );
+ }
+ }
+
+ return compare;
+ }
+
+
+ /**
+ * Return the Id to be used in the comparison
+ * @param edge
+ * @return
+ */
+ protected abstract Id getId(final MarkedEdge edge);
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/OrderedComparator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/OrderedComparator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/OrderedComparator.java
new file mode 100644
index 0000000..003ed36
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/OrderedComparator.java
@@ -0,0 +1,52 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators;
+
+
+import java.util.Comparator;
+
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+
+
+/**
+ * Comparator that will compare in reverse or forward order based on the order type specified.
+ *
+ * Assumes descending uses the default order. If ASCENDING, the result of the comparator will be reversed
+ */
+public class OrderedComparator<T> implements Comparator<T> {
+
+
+ private final int invert;
+ private final Comparator<T> delegate;
+
+
+ public OrderedComparator( final Comparator<T> delegate, final SearchByEdgeType.Order order ) {
+ this.invert = order == SearchByEdgeType.Order.DESCENDING ? 1 : -1;
+ this.delegate = delegate;
+ }
+
+
+ @Override
+ public int compare( final T o1, final T o2 ) {
+ return delegate.compare( o1, o2 ) * invert;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparator.java
new file mode 100644
index 0000000..f067006
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparator.java
@@ -0,0 +1,42 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators;
+
+
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ *
+ * Comparator that uses source Id for comparisons. Newer times will be "greater". Newer uuids will be first.
+ *
+ */
+public class SourceDirectedEdgeDescendingComparator extends DirectedEdgeDescendingComparator {
+
+ public static final SourceDirectedEdgeDescendingComparator INSTANCE = new SourceDirectedEdgeDescendingComparator();
+
+ @Override
+ protected Id getId( final MarkedEdge edge ) {
+ return edge.getSourceNode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparator.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparator.java
new file mode 100644
index 0000000..115a874
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparator.java
@@ -0,0 +1,42 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators;
+
+
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+
+/**
+ *
+ * Comparator that uses source Id for comparisons. Newer time uuids will be first.
+ *
+ */
+public class TargetDirectedEdgeDescendingComparator extends DirectedEdgeDescendingComparator {
+
+ public static final TargetDirectedEdgeDescendingComparator INSTANCE = new TargetDirectedEdgeDescendingComparator();
+
+ @Override
+ protected Id getId( final MarkedEdge edge ) {
+ return edge.getTargetNode();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeRowKeySerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeRowKeySerializer.java
new file mode 100644
index 0000000..f1ae90b
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeRowKeySerializer.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize;
+
+
+import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+
+
+/**
+ * Class to perform serialization for row keys from edges
+ */
+
+public class EdgeRowKeySerializer implements CompositeFieldSerializer<EdgeRowKey> {
+
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ @Override
+ public void toComposite( final CompositeBuilder builder, final EdgeRowKey key ) {
+
+ //add the row id to the composite
+ ID_SER.toComposite( builder, key.sourceId );
+ builder.addString( key.edgeType );
+ ID_SER.toComposite( builder, key.targetId );
+ builder.addLong( key.shardId );
+ }
+
+
+ @Override
+ public EdgeRowKey fromComposite( final CompositeParser composite ) {
+
+ final Id sourceId = ID_SER.fromComposite( composite );
+ final String edgeType = composite.readString();
+ final Id targetId = ID_SER.fromComposite( composite );
+ final long shard = composite.readLong();
+
+ return new EdgeRowKey( sourceId, edgeType, targetId, shard );
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java
new file mode 100644
index 0000000..590cf35
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeSerializer.java
@@ -0,0 +1,77 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.usergrid.persistence.core.astyanax.IdColDynamicCompositeSerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.google.common.base.Preconditions;
+import com.netflix.astyanax.model.DynamicComposite;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+import com.netflix.astyanax.serializers.LongSerializer;
+
+
+/**
+ * Serializes to a source->target edge Note that we cannot set the edge type on de-serialization. Only the target
+ * Id and version.
+ */
+public class EdgeSerializer extends AbstractSerializer<DirectedEdge> {
+
+ private static final IdColDynamicCompositeSerializer ID_COL_SERIALIZER = IdColDynamicCompositeSerializer.get();
+ private static final LongSerializer LONG_SERIALIZER = LongSerializer.get();
+
+
+ @Override
+ public ByteBuffer toByteBuffer( final DirectedEdge edge ) {
+
+ DynamicComposite composite = new DynamicComposite();
+
+ composite.addComponent( edge.timestamp, LONG_SERIALIZER );
+
+ ID_COL_SERIALIZER.toComposite( composite, edge.id );
+
+ return composite.serialize();
+ }
+
+
+ @Override
+ public DirectedEdge fromByteBuffer( final ByteBuffer byteBuffer ) {
+ DynamicComposite composite = DynamicComposite.fromByteBuffer( byteBuffer );
+
+ Preconditions.checkArgument( composite.size() == 3, "Composite should have 3 elements" );
+
+
+ //return the version
+ final long timestamp = composite.get( 0, LONG_SERIALIZER );
+
+
+ //parse our id
+ final Id id = ID_COL_SERIALIZER.fromComposite( composite, 1 );
+
+
+ return new DirectedEdge( id, timestamp );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeShardRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeShardRowKeySerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeShardRowKeySerializer.java
new file mode 100644
index 0000000..6b1c4e9
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/EdgeShardRowKeySerializer.java
@@ -0,0 +1,103 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize;
+
+
+import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+
+
+public class EdgeShardRowKeySerializer implements CompositeFieldSerializer<DirectedEdgeMeta> {
+
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+ public static final EdgeShardRowKeySerializer INSTANCE = new EdgeShardRowKeySerializer();
+
+
+ @Override
+ public void toComposite( final CompositeBuilder builder, final DirectedEdgeMeta meta ) {
+
+
+ final DirectedEdgeMeta.NodeMeta[] nodeMeta = meta.getNodes();
+
+ //add the stored value
+ builder.addInteger( meta.getType().getStorageValue() );
+
+ final int length = nodeMeta.length;
+
+ builder.addInteger( length );
+
+
+ for ( DirectedEdgeMeta.NodeMeta node : nodeMeta ) {
+ ID_SER.toComposite( builder, node.getId() );
+ builder.addInteger( node.getNodeType().getStorageValue() );
+ }
+
+ final String[] edgeTypes = meta.getTypes();
+
+ builder.addInteger( edgeTypes.length );
+
+ for ( String type : edgeTypes ) {
+ builder.addString( type );
+ }
+ }
+
+
+ @Override
+ public DirectedEdgeMeta fromComposite( final CompositeParser composite ) {
+
+
+ final int storageType = composite.readInteger();
+
+ final DirectedEdgeMeta.MetaType metaType = DirectedEdgeMeta.MetaType.fromStorage( storageType );
+
+ final int idLength = composite.readInteger();
+
+ final DirectedEdgeMeta.NodeMeta[] nodePairs = new DirectedEdgeMeta.NodeMeta[idLength];
+
+
+ for ( int i = 0; i < idLength; i++ ) {
+ final Id sourceId = ID_SER.fromComposite( composite );
+
+ final NodeType type = NodeType.get( composite.readInteger() );
+
+ nodePairs[i] = new DirectedEdgeMeta.NodeMeta( sourceId, type );
+ }
+
+
+ final int length = composite.readInteger();
+
+ String[] types = new String[length];
+
+ for ( int i = 0; i < length; i++ ) {
+ types[i] = composite.readString();
+ }
+
+ return DirectedEdgeMeta.fromStorage( metaType, nodePairs, types );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowSerializer.java
new file mode 100644
index 0000000..8376ef1
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowSerializer.java
@@ -0,0 +1,63 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize;
+
+
+import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+
+
+/**
+ * Class to perform serialization for row keys from edges
+ */
+public class RowSerializer implements CompositeFieldSerializer<RowKey> {
+
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ @Override
+ public void toComposite( final CompositeBuilder builder, final RowKey key ) {
+
+ //add the row id to the composite
+ ID_SER.toComposite( builder, key.nodeId );
+
+ builder.addString( key.edgeType );
+ builder.addLong( key.shardId );
+ }
+
+
+ @Override
+ public RowKey fromComposite( final CompositeParser composite ) {
+
+ final Id id = ID_SER.fromComposite( composite );
+ final String edgeType = composite.readString();
+ final long shard = composite.readLong();
+
+
+ return new RowKey( id, edgeType, shard );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowTypeSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowTypeSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowTypeSerializer.java
new file mode 100644
index 0000000..a67c469
--- /dev/null
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/serialize/RowTypeSerializer.java
@@ -0,0 +1,62 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize;
+
+
+import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
+import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKeyType;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+
+
+public class RowTypeSerializer implements CompositeFieldSerializer<RowKeyType> {
+
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ @Override
+ public void toComposite( final CompositeBuilder builder, final RowKeyType keyType ) {
+
+ //add the row id to the composite
+ ID_SER.toComposite( builder, keyType.nodeId );
+
+ builder.addString( keyType.edgeType );
+ builder.addString( keyType.idType );
+
+ builder.addLong( keyType.shardId );
+ }
+
+
+ @Override
+ public RowKeyType fromComposite( final CompositeParser composite ) {
+
+ final Id id = ID_SER.fromComposite( composite );
+ final String edgeType = composite.readString();
+ final String idType = composite.readString();
+ final long shard = composite.readLong();
+
+ return new RowKeyType( id, edgeType, idType, shard);
+ }
+}
[4/5] Updated OrderedMerge to use a faster implementation at runtime.
After initialization,
it's an O(1) emit operation as long as our produces are fast enough.
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
index d434db7..0a6ecfa 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/GraphFig.java
@@ -35,8 +35,15 @@ public interface GraphFig extends GuicyFig {
public static final String REPAIR_CONCURRENT_SIZE = "usergrid.graph.repair.concurrent.size";
+ /**
+ * The size of the shards. This is approximate, and should be set lower than what you would like your max to be
+ */
public static final String SHARD_SIZE = "usergrid.graph.shard.size";
+
+ /**
+ * Number of shards we can cache.
+ */
public static final String SHARD_CACHE_SIZE = "usergrid.graph.shard.cache.size";
@@ -45,17 +52,33 @@ public interface GraphFig extends GuicyFig {
*/
public static final String SHARD_CACHE_TIMEOUT = "usergrid.graph.shard.cache.timeout";
+ /**
+ * Number of worker threads to refresh the cache
+ */
public static final String SHARD_CACHE_REFRESH_WORKERS = "usergrid.graph.shard.refresh.worker.count";
- public static final String SHARD_REPAIR_CHANCE = "usergrid.graph.shard.repair.chance";
+ /**
+ * The size of the worker count for shard auditing
+ */
+ public static final String SHARD_AUDIT_QUEUE_SIZE = "usergrid.graph.shard.audit.worker.queue.size";
/**
- * The minimum amount of time than can occur (in millis) between shard allocation. Must be at least 2x the cache timeout.
+ * The size of the worker count for shard auditing
+ */
+ public static final String SHARD_AUDIT_WORKERS = "usergrid.graph.shard.audit.worker.count";
+
+
+ public static final String SHARD_REPAIR_CHANCE = "usergrid.graph.shard.repair.chance";
+
+
+ /**
+ * The minimum amount of time than can occur (in millis) between shard allocation and compaction. Must be at least 2x the cache
+ * timeout. Set to 2.5x the cache timeout to be safe
*
- * Note that you should also pad this for node clock drift. A good value for this would be 2x the shard cache timeout + 30 seconds,
- * assuming you have NTP and allow a max drift of 30 seconds
+ * Note that you should also pad this for node clock drift. A good value for this would be 2x the shard cache
+ * timeout + 30 seconds, assuming you have NTP and allow a max drift of 30 seconds
*/
public static final String SHARD_MIN_DELTA = "usergrid.graph.shard.min.delta";
@@ -67,26 +90,23 @@ public interface GraphFig extends GuicyFig {
public static final String COUNTER_WRITE_FLUSH_QUEUE_SIZE = "usergrid.graph.shard.counter.queue.size";
-
-
@Default("1000")
@Key(SCAN_PAGE_SIZE)
int getScanPageSize();
-
@Default("5")
@Key(REPAIR_CONCURRENT_SIZE)
int getRepairConcurrentSize();
@Default( ".10" )
- @Key( SHARD_REPAIR_CHANCE )
+ @Key( SHARD_REPAIR_CHANCE )
double getShardRepairChance();
- @Default("500000")
- @Key(SHARD_SIZE)
+ @Default( "500000" )
+ @Key( SHARD_SIZE )
long getShardSize();
@@ -95,31 +115,40 @@ public interface GraphFig extends GuicyFig {
long getShardCacheTimeout();
@Default("60000")
- @Key( SHARD_MIN_DELTA )
+ @Key(SHARD_MIN_DELTA)
long getShardMinDelta();
- @Default( "250000" )
- @Key( SHARD_CACHE_SIZE )
+ @Default("250000")
+ @Key(SHARD_CACHE_SIZE)
long getShardCacheSize();
- @Default( "2" )
- @Key( SHARD_CACHE_REFRESH_WORKERS )
+ @Default("2")
+ @Key(SHARD_CACHE_REFRESH_WORKERS)
int getShardCacheRefreshWorkerCount();
- @Default( "10000" )
- @Key( COUNTER_WRITE_FLUSH_COUNT )
+ @Default( "10" )
+ @Key( SHARD_AUDIT_WORKERS )
+ int getShardAuditWorkerCount();
+
+ @Default( "1000" )
+ @Key( SHARD_AUDIT_QUEUE_SIZE )
+ int getShardAuditWorkerQueueSize();
+
+
+ @Default("10000")
+ @Key(COUNTER_WRITE_FLUSH_COUNT)
long getCounterFlushCount();
- @Default( "30000" )
- @Key( COUNTER_WRITE_FLUSH_INTERVAL )
+ @Default("30000")
+ @Key(COUNTER_WRITE_FLUSH_INTERVAL)
long getCounterFlushInterval();
- @Default( "1000" )
- @Key(COUNTER_WRITE_FLUSH_QUEUE_SIZE )
+ @Default("1000")
+ @Key(COUNTER_WRITE_FLUSH_QUEUE_SIZE)
int getCounterFlushQueueSize();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
index 7e589f2..114440f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdge.java
@@ -65,4 +65,10 @@ public interface SearchByEdge {
*/
Optional<Edge> last();
+ /**
+ * Get the sort order
+ * @return
+ */
+ SearchByEdgeType.Order getOrder();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
index 29cc3f5..749130b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/SearchByEdgeType.java
@@ -59,4 +59,20 @@ public interface SearchByEdgeType {
*/
Optional<Edge> last();
+ /**
+ * Get the direction we're seeking
+ * @return
+ */
+ Order getOrder();
+
+
+ /**
+ * Options for ordering. By default, we want to perform descending for common use cases and read speed. This is our our data
+ * is optimized in cassandra
+ */
+ public enum Order {
+ DESCENDING,
+ ASCENDING
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
index 64f0fbb..f0e954b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/GraphModule.java
@@ -48,6 +48,7 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardS
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardAllocation;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupCompaction;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.count.NodeShardApproximationImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.count.NodeShardCounterSerialization;
@@ -55,6 +56,7 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.count.Node
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.EdgeShardSerializationImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.NodeShardAllocationImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.NodeShardCacheImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardGroupCompactionImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardedEdgeSerializationImpl;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.SizebasedEdgeColumnFamilies;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.SizebasedEdgeShardStrategy;
@@ -119,6 +121,8 @@ public class GraphModule extends AbstractModule {
bind( EdgeColumnFamilies.class ).to( SizebasedEdgeColumnFamilies.class );
+ bind( ShardGroupCompaction.class).to( ShardGroupCompactionImpl.class);
+
/**
* Bind our implementation
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/MergedProxy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/MergedProxy.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/MergedProxy.java
deleted file mode 100644
index 20bc637..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/guice/MergedProxy.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.graph.guice;
-
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-import com.google.inject.BindingAnnotation;
-
-
-@Retention(RetentionPolicy.RUNTIME)
-@Target({ ElementType.FIELD, ElementType.PARAMETER, ElementType.METHOD })
-@BindingAnnotation
-public @interface MergedProxy {}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
index 12192fc..9fcb816 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleMarkedEdge.java
@@ -70,7 +70,6 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
return false;
}
-
return true;
}
@@ -90,3 +89,4 @@ public class SimpleMarkedEdge extends SimpleEdge implements MarkedEdge {
"} " + super.toString();
}
}
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
index e8971f6..d40efc0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdge.java
@@ -23,10 +23,12 @@ package org.apache.usergrid.persistence.graph.impl;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.SearchByEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
/**
@@ -40,6 +42,7 @@ public class SimpleSearchByEdge implements SearchByEdge {
private final String type;
private final long maxTimestamp;
private final Optional<Edge> last;
+ private final SearchByEdgeType.Order order;
/**
@@ -50,17 +53,20 @@ public class SimpleSearchByEdge implements SearchByEdge {
* @param maxTimestamp The maximum timestamp to seek from
* @param last The value to start seeking from. Must be >= this value
*/
- public SimpleSearchByEdge( final Id sourceNode, final String type, final Id targetNode, final long maxTimestamp, final Edge last ) {
+ public SimpleSearchByEdge( final Id sourceNode, final String type, final Id targetNode, final long maxTimestamp, final SearchByEdgeType.Order order, final Edge last ) {
+
ValidationUtils.verifyIdentity(sourceNode);
ValidationUtils.verifyIdentity(targetNode);
ValidationUtils.verifyString( type, "type" );
GraphValidation.validateTimestamp( maxTimestamp, "maxTimestamp" );
+ Preconditions.checkNotNull(order, "order must not be null");
this.sourceNode = sourceNode;
this.targetNode = targetNode;
this.type = type;
this.maxTimestamp = maxTimestamp;
+ this.order = order;
this.last = Optional.fromNullable(last);
}
@@ -93,4 +99,10 @@ public class SimpleSearchByEdge implements SearchByEdge {
public Optional<Edge> last() {
return last;
}
+
+
+ @Override
+ public SearchByEdgeType.Order getOrder() {
+ return order;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
index 9e7dcde..6bc8b1b 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByEdgeType.java
@@ -27,6 +27,7 @@ import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Optional;
+import com.google.common.base.Preconditions;
/**
@@ -39,6 +40,7 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
private final String type;
private final long maxTimestamp;
private final Optional<Edge> last;
+ private final Order order;
/**
@@ -46,9 +48,14 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
* @param node The node to search from
* @param type The edge type
* @param maxTimestamp The maximum timestamp to return
+ * @param order The order order. Descending is most efficient
* @param last The value to start seeking from. Must be >= this value
+ * @param order
*/
- public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Edge last ) {
+ public SimpleSearchByEdgeType( final Id node, final String type, final long maxTimestamp, final Order order, final Edge last
+ ) {
+
+ Preconditions.checkNotNull( order, "order is required");
ValidationUtils.verifyIdentity(node);
ValidationUtils.verifyString( type, "type" );
GraphValidation.validateTimestamp( maxTimestamp, "maxTimestamp" );
@@ -57,6 +64,7 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
this.node = node;
this.type = type;
this.maxTimestamp = maxTimestamp;
+ this.order = order;
this.last = Optional.fromNullable(last);
}
@@ -86,6 +94,12 @@ public class SimpleSearchByEdgeType implements SearchByEdgeType{
@Override
+ public Order getOrder() {
+ return order;
+ }
+
+
+ @Override
public boolean equals( final Object o ) {
if ( this == o ) {
return true;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
index 4249ae7..4b73347 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/SimpleSearchByIdType.java
@@ -44,8 +44,8 @@ public class SimpleSearchByIdType extends SimpleSearchByEdgeType implements Sear
* @param last The value to start seeking from. Must be >= this value
*/
- public SimpleSearchByIdType( final Id node, final String type, final long maxTimestamp, final String idType, final Edge last ) {
- super( node, type, maxTimestamp, last );
+ public SimpleSearchByIdType( final Id node, final String type, final long maxTimestamp, final Order order, final String idType, final Edge last ) {
+ super( node, type, maxTimestamp, order, last );
ValidationUtils.verifyString( idType, "idType" );
this.idType = idType;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
index 5be3541..0137ba4 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairImpl.java
@@ -32,14 +32,13 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action1;
@@ -116,7 +115,7 @@ public class EdgeDeleteRepairImpl implements EdgeDeleteRepair {
final SimpleSearchByEdge search =
new SimpleSearchByEdge( edge.getSourceNode(), edge.getType(), edge.getTargetNode(),
- edge.getTimestamp(), null );
+ edge.getTimestamp(), SearchByEdgeType.Order.DESCENDING, null );
return serialization.getEdgeVersions( scope, search );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
index 055b867..7e09eca 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairImpl.java
@@ -33,7 +33,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchIdType;
import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
@@ -46,7 +46,6 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action1;
@@ -286,7 +285,7 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesToTargetBySourceType( scope,
- new SimpleSearchByIdType( nodeId, edgeType, maxTimestamp, subType, null ) );
+ new SimpleSearchByIdType( nodeId, edgeType, maxTimestamp, SearchByEdgeType.Order.DESCENDING, subType, null ) );
}
} );
}
@@ -332,7 +331,7 @@ public class EdgeMetaRepairImpl implements EdgeMetaRepair {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageEdgeSerialization.getEdgesFromSourceByTargetType( scope,
- new SimpleSearchByIdType( nodeId, edgeType, maxTimestamp, subType, null ) );
+ new SimpleSearchByIdType( nodeId, edgeType, maxTimestamp, SearchByEdgeType.Order.DESCENDING, subType, null ) );
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
index 962da21..2be6c55 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/impl/stage/NodeDeleteListenerImpl.java
@@ -33,8 +33,8 @@ import org.apache.usergrid.persistence.core.rx.ObservableIterator;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.SearchEdgeType;
-import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
import org.apache.usergrid.persistence.graph.serialization.EdgeMetadataSerialization;
@@ -46,7 +46,6 @@ import com.google.common.base.Optional;
import com.google.inject.Inject;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import rx.Observable;
import rx.functions.Action0;
@@ -160,7 +159,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesToTarget( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, null ) );
+ new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, null ) );
}
} );
}
@@ -177,7 +176,7 @@ public class NodeDeleteListenerImpl implements NodeDeleteListener {
@Override
protected Iterator<MarkedEdge> getIterator() {
return storageSerialization.getEdgesFromSource( scope,
- new SimpleSearchByEdgeType( node, edgeType, maxVersion, null ) );
+ new SimpleSearchByEdgeType( node, edgeType, maxVersion, SearchByEdgeType.Order.DESCENDING, null ) );
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
index 92f2548..6bb467f 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/DirectedEdgeMeta.java
@@ -28,6 +28,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import java.util.UUID;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.MarkedEdge;
@@ -38,6 +39,8 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByIdType;
import org.apache.usergrid.persistence.model.entity.Id;
+import com.netflix.astyanax.MutationBatch;
+
/**
* A bean to define directed edge meta data. This is used to encapsulate the meta data around a source or target node,
@@ -151,14 +154,52 @@ public abstract class DirectedEdgeMeta {
}
+ @Override
+ public String toString() {
+ return "DirectedEdgeMeta{" +
+ "nodes=" + Arrays.toString( nodes ) +
+ ", types=" + Arrays.toString( types ) +
+ '}';
+ }
+
+
/**
* Given the edge serialization, load all shard in the shard group
*/
public abstract Iterator<MarkedEdge> loadEdges( final ShardedEdgeSerialization serialization,
final EdgeColumnFamilies edgeColumnFamilies,
final ApplicationScope scope, final Collection<Shard> shards,
- final long maxValue );
+ final long maxValue, final SearchByEdgeType.Order order );
+
+
+ /**
+ * Write the edge for this meta data to the target edge
+ * @param shardedEdgeSerialization
+ * @param edgeColumnFamilies
+ * @param scope
+ * @param targetShard
+ * @param edge
+ * @param timestamp The timestamp on the operation
+ * @return
+ */
+ public abstract MutationBatch writeEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard targetShard, final MarkedEdge edge, final UUID timestamp );
+
+ /**
+ * Delete the edge for this meta data from the shard
+ * @param shardedEdgeSerialization
+ * @param edgeColumnFamilies
+ * @param scope
+ * @param sourceShard
+ * @param edge
+ * @param timestamp The timestamp on the operation
+ * @return
+ */
+ public abstract MutationBatch deleteEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard sourceShard, final MarkedEdge edge, final UUID timestamp );
/**
@@ -225,19 +266,39 @@ public abstract class DirectedEdgeMeta {
@Override
public Iterator<MarkedEdge> loadEdges( final ShardedEdgeSerialization serialization,
final EdgeColumnFamilies edgeColumnFamilies,
- final ApplicationScope scope, final Collection<Shard> shards,
- final long maxValue ) {
+ final ApplicationScope scope, final Collection<Shard> shards,
+ final long maxValue, final SearchByEdgeType.Order order ) {
final Id sourceId = nodes[0].id;
final String edgeType = types[0];
- final SearchByEdgeType search = new SimpleSearchByEdgeType( sourceId, edgeType, maxValue, null );
+ final SearchByEdgeType search = new SimpleSearchByEdgeType( sourceId, edgeType, maxValue, order, null);
return serialization.getEdgesFromSource( edgeColumnFamilies, scope, search, shards );
}
@Override
+ public MutationBatch writeEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard targetShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization
+ .writeEdgeFromSource( edgeColumnFamilies, scope, edge, Collections.singleton( targetShard ),
+ this, timestamp );
+ }
+
+
+ @Override
+ public MutationBatch deleteEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard sourceShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization
+ .deleteEdgeFromSource( edgeColumnFamilies, scope, edge, Collections.singleton( sourceShard ),
+ this, timestamp );
+ }
+
+
+ @Override
public MetaType getType() {
return MetaType.SOURCE;
}
@@ -264,19 +325,39 @@ public abstract class DirectedEdgeMeta {
@Override
public Iterator<MarkedEdge> loadEdges( final ShardedEdgeSerialization serialization,
- final EdgeColumnFamilies edgeColumnFamilies,
- final ApplicationScope scope, final Collection<Shard>shards,
- final long maxValue ) {
-//
+ final EdgeColumnFamilies edgeColumnFamilies,
+ final ApplicationScope scope, final Collection<Shard> shards,
+ final long maxValue, final SearchByEdgeType.Order order ) {
+ //
final Id sourceId = nodes[0].id;
final String edgeType = types[0];
final String targetType = types[1];
final SearchByIdType search =
- new SimpleSearchByIdType( sourceId, edgeType, maxValue, targetType, null );
+ new SimpleSearchByIdType( sourceId, edgeType, maxValue, order, targetType, null );
+
+ return serialization.getEdgesFromSourceByTargetType( edgeColumnFamilies, scope, search, shards );
+ }
- return serialization.getEdgesFromSourceByTargetType( edgeColumnFamilies, scope, search, shards);
+
+
+
+ @Override
+ public MutationBatch writeEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard targetShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization.writeEdgeFromSourceWithTargetType( edgeColumnFamilies, scope, edge,
+ Collections.singleton( targetShard ), this, timestamp );
+ }
+
+
+ @Override
+ public MutationBatch deleteEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard sourceShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization.deleteEdgeFromSourceWithTargetType( edgeColumnFamilies, scope, edge,
+ Collections.singleton( sourceShard ), this, timestamp );
}
@@ -304,20 +385,40 @@ public abstract class DirectedEdgeMeta {
@Override
public Iterator<MarkedEdge> loadEdges( final ShardedEdgeSerialization serialization,
final EdgeColumnFamilies edgeColumnFamilies,
- final ApplicationScope scope, final Collection<Shard> shards,
- final long maxValue ) {
+ final ApplicationScope scope, final Collection<Shard> shards,
+ final long maxValue, final SearchByEdgeType.Order order ) {
final Id targetId = nodes[0].id;
final String edgeType = types[0];
- final SearchByEdgeType search = new SimpleSearchByEdgeType( targetId, edgeType, maxValue, null );
+ final SearchByEdgeType search = new SimpleSearchByEdgeType( targetId, edgeType, maxValue, order, null);
return serialization.getEdgesToTarget( edgeColumnFamilies, scope, search, shards );
}
@Override
+ public MutationBatch writeEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard targetShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization
+ .writeEdgeToTarget( edgeColumnFamilies, scope, edge, Collections.singleton( targetShard ),
+ this, timestamp );
+ }
+
+
+ @Override
+ public MutationBatch deleteEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard sourceShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization
+ .deleteEdgeToTarget( edgeColumnFamilies, scope, edge, Collections.singleton( sourceShard ),
+ this, timestamp );
+ }
+
+
+ @Override
public MetaType getType() {
return MetaType.TARGET;
}
@@ -339,11 +440,13 @@ public abstract class DirectedEdgeMeta {
private static DirectedEdgeMeta fromTargetNodeSourceType( final NodeMeta[] nodes, final String[] types ) {
return new DirectedEdgeMeta( nodes, types ) {
+
@Override
public Iterator<MarkedEdge> loadEdges( final ShardedEdgeSerialization serialization,
final EdgeColumnFamilies edgeColumnFamilies,
final ApplicationScope scope, final Collection<Shard> shards,
- final long maxValue ) {
+ final long maxValue, final SearchByEdgeType.Order order ) {
+
final Id targetId = nodes[0].id;
final String edgeType = types[0];
@@ -351,9 +454,27 @@ public abstract class DirectedEdgeMeta {
final SearchByIdType search =
- new SimpleSearchByIdType( targetId, edgeType, maxValue, sourceType, null );
+ new SimpleSearchByIdType( targetId, edgeType, maxValue, order, sourceType, null );
+
+ return serialization.getEdgesToTargetBySourceType( edgeColumnFamilies, scope, search, shards );
+ }
+
- return serialization.getEdgesToTargetBySourceType( edgeColumnFamilies, scope, search, shards);
+ @Override
+ public MutationBatch writeEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard targetShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization.writeEdgeToTargetWithSourceType( edgeColumnFamilies, scope, edge,
+ Collections.singleton( targetShard ), this, timestamp );
+ }
+
+
+ @Override
+ public MutationBatch deleteEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard sourceShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization.deleteEdgeToTargetWithSourceType( edgeColumnFamilies, scope, edge,
+ Collections.singleton( sourceShard ), this, timestamp );
}
@@ -385,18 +506,37 @@ public abstract class DirectedEdgeMeta {
@Override
public Iterator<MarkedEdge> loadEdges( final ShardedEdgeSerialization serialization,
final EdgeColumnFamilies edgeColumnFamilies,
- final ApplicationScope scope, final Collection<Shard> shards,
- final long maxValue ) {
+ final ApplicationScope scope, final Collection<Shard> shards,
+ final long maxValue, final SearchByEdgeType.Order order ) {
final Id sourceId = nodes[0].id;
final Id targetId = nodes[1].id;
final String edgeType = types[0];
final SimpleSearchByEdge search =
- new SimpleSearchByEdge( sourceId, edgeType, targetId, maxValue, null );
+ new SimpleSearchByEdge( sourceId, edgeType, targetId, maxValue, order, null );
+
+ return serialization.getEdgeVersions( edgeColumnFamilies, scope, search, shards );
+ }
- return serialization.getEdgeVersions( edgeColumnFamilies, scope, search, shards);
+ @Override
+ public MutationBatch writeEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard targetShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization
+ .writeEdgeVersions( edgeColumnFamilies, scope, edge, Collections.singleton( targetShard ),
+ this, timestamp );
+ }
+
+
+ @Override
+ public MutationBatch deleteEdge( final ShardedEdgeSerialization shardedEdgeSerialization,
+ final EdgeColumnFamilies edgeColumnFamilies, final ApplicationScope scope,
+ final Shard sourceShard, final MarkedEdge edge, final UUID timestamp ) {
+ return shardedEdgeSerialization
+ .deleteEdgeVersions( edgeColumnFamilies, scope, edge, Collections.singleton( sourceShard ),
+ this, timestamp );
}
@@ -408,16 +548,15 @@ public abstract class DirectedEdgeMeta {
}
-
/**
* Create a directed edge from the stored meta data
+ *
* @param metaType The meta type stored
* @param nodes The metadata of the nodes
* @param types The types in the meta data
- *
- *
*/
- public static DirectedEdgeMeta fromStorage( final MetaType metaType, final NodeMeta[] nodes, final String[] types ) {
+ public static DirectedEdgeMeta fromStorage( final MetaType metaType, final NodeMeta[] nodes,
+ final String[] types ) {
switch ( metaType ) {
case SOURCE:
return fromSourceNode( nodes, types );
@@ -428,7 +567,7 @@ public abstract class DirectedEdgeMeta {
case TARGETSOURCE:
return fromTargetNodeSourceType( nodes, types );
case VERSIONS:
- return fromEdge(nodes, types);
+ return fromEdge( nodes, types );
default:
throw new UnsupportedOperationException( "No supported meta type found" );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKey.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKey.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKey.java
index 9bd9937..74f7ffc 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKey.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKey.java
@@ -42,4 +42,14 @@ public class RowKey {
this.edgeType = edgeType;
this.shardId = shardId;
}
+
+
+ @Override
+ public String toString() {
+ return "RowKey{" +
+ "nodeId=" + nodeId +
+ ", edgeType='" + edgeType + '\'' +
+ ", shardId=" + shardId +
+ '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKeyType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKeyType.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKeyType.java
index 6e69bbf..3368c40 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKeyType.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/RowKeyType.java
@@ -57,5 +57,10 @@ public class RowKeyType extends RowKey {
}
-
+ @Override
+ public String toString() {
+ return "RowKeyType{" +
+ "idType='" + idType + '\'' +
+ "} " + super.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
index 38fe51c..9ca6cbe 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/Shard.java
@@ -137,6 +137,7 @@ public class Shard implements Comparable<Shard> {
return "Shard{" +
"shardIndex=" + shardIndex +
", createdTime=" + createdTime +
- "} ";
+ ", compacted=" + compacted +
+ '}';
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroup.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroup.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroup.java
index 70569fd..11bf7a4 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroup.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroup.java
@@ -20,10 +20,15 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import com.google.common.base.Preconditions;
@@ -36,6 +41,7 @@ import com.google.common.base.Preconditions;
*/
public class ShardEntryGroup {
+ private static final Logger LOG = LoggerFactory.getLogger( ShardEntryGroup.class );
private List<Shard> shards;
@@ -45,6 +51,8 @@ public class ShardEntryGroup {
private Shard compactionTarget;
+ private Shard rootShard;
+
/**
* The max delta we accept in milliseconds for create time to be considered a member of this group
@@ -85,10 +93,13 @@ public class ShardEntryGroup {
//shard is not compacted, or it's predecessor isn't, we should include it in this group
if ( !minShard.isCompacted() ) {
addShardInternal( shard );
+
return true;
}
+
+
return false;
}
@@ -124,10 +135,26 @@ public class ShardEntryGroup {
* Get the entries that we should read from.
*/
public Collection<Shard> getReadShards() {
- return shards;
+
+
+ final Shard staticShard = getRootShard();
+ final Shard compactionTarget = getCompactionTarget();
+
+
+
+ if(compactionTarget != null){
+ LOG.debug( "Returning shards {} and {} as read shards", compactionTarget, staticShard );
+ return Arrays.asList( compactionTarget, staticShard );
+ }
+
+
+ LOG.debug( "Returning shards {} read shard", staticShard );
+ return Collections.singleton( staticShard );
}
+
+
/**
* Get the entries, with the max shard time being first. We write to all shards until they're migrated
*/
@@ -138,11 +165,22 @@ public class ShardEntryGroup {
* adding data to other shards
*/
if ( !isTooSmallToCompact() && shouldCompact( currentTime ) ) {
- return Collections.singleton( getCompactionTarget() );
+
+ final Shard compactionTarget = getCompactionTarget();
+
+ LOG.debug( "Returning shard {} as write shard", compactionTarget);
+
+ return Collections.singleton( compactionTarget );
+
}
+ final Shard staticShard = getRootShard();
+
+
+ LOG.debug( "Returning shard {} as write shard", staticShard);
+
+ return Collections.singleton( staticShard );
- return shards;
}
@@ -155,6 +193,24 @@ public class ShardEntryGroup {
/**
+ * Get the root shard that was created in this group
+ * @return
+ */
+ private Shard getRootShard(){
+ if(rootShard != null){
+ return rootShard;
+ }
+
+ final Shard rootCandidate = shards.get( shards.size() -1 );
+
+ if(rootCandidate.isCompacted()){
+ rootShard = rootCandidate;
+ }
+
+ return rootShard;
+ }
+
+ /**
* Get the shard all compactions should write to. Null indicates we cannot find a shard that could be used as a
* compaction target. Note that this shard may not have surpassed the delta yet You should invoke "shouldCompact"
* first to ensure all criteria are met before initiating compaction
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompaction.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompaction.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompaction.java
index bf4d3c9..4fe1a63 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompaction.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompaction.java
@@ -22,38 +22,32 @@
package org.apache.usergrid.persistence.graph.serialization.impl.shard;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.Set;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import rx.Observable;
+import com.google.common.util.concurrent.ListenableFuture;
/**
* Defines tasks for running compaction
- *
- *
*/
public interface ShardGroupCompaction {
-
- /**
- * Execute the compaction task. Will return the number of edges that have
- * @param group The shard entry group to compact
- * @return The shards that were compacted
- */
- public Set<Shard> compact(final ApplicationScope scope, final DirectedEdgeMeta edgeMeta, final ShardEntryGroup group);
-
/**
- * Possibly audit the shard entry group. This is asynchronous and returns immediately
- * @param group
- * @return
+ * Possibly audit the shard entry group. This is asynchronous and returns the future that will
+ * report the operations performed (if any) upon completion.
+ *
+ * @return A ListenableFuture with the result. Note that some
*/
- public AuditResult evaluateShardGroup( final ApplicationScope scope, final DirectedEdgeMeta edgeMeta,
- final ShardEntryGroup group );
+ public ListenableFuture<AuditResult> evaluateShardGroup( final ApplicationScope scope,
+ final DirectedEdgeMeta edgeMeta,
+ final ShardEntryGroup group );
- public enum AuditResult{
+ public enum AuditResult {
/**
* We didn't check this shard
*/
@@ -68,11 +62,10 @@ public interface ShardGroupCompaction {
*/
CHECKED_CREATED,
- /**
+ COMPACTED, /**
* The shard group is already compacting
*/
COMPACTING
}
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
index ed3daaf..749416c 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
@@ -39,10 +39,9 @@ import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.graph.GraphFig;
-import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.EdgeShardRowKeySerializer;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.EdgeShardRowKeySerializer;
import org.apache.usergrid.persistence.model.entity.Id;
import com.google.common.base.Preconditions;
@@ -51,7 +50,6 @@ import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
import com.netflix.astyanax.connectionpool.OperationResult;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.CompositeBuilder;
@@ -127,7 +125,10 @@ public class NodeShardCounterSerializationImpl implements NodeShardCounterSerial
}
//column not found, return 0
catch ( RuntimeException re ) {
- if(re.getCause().getCause() instanceof NotFoundException) {
+
+ final Throwable cause = re.getCause();
+
+ if(cause != null && cause.getCause() instanceof NotFoundException) {
return 0;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeRowKeySerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeRowKeySerializer.java
deleted file mode 100644
index 90b264c..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeRowKeySerializer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
-
-
-import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
-import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-
-
-/**
- * Class to perform serialization for row keys from edges
- */
-
-public class EdgeRowKeySerializer implements CompositeFieldSerializer<EdgeRowKey> {
-
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
-
- @Override
- public void toComposite( final CompositeBuilder builder, final EdgeRowKey key ) {
-
- //add the row id to the composite
- ID_SER.toComposite( builder, key.sourceId );
- builder.addString( key.edgeType );
- ID_SER.toComposite( builder, key.targetId );
- builder.addLong( key.shardId );
- }
-
-
- @Override
- public EdgeRowKey fromComposite( final CompositeParser composite ) {
-
- final Id sourceId = ID_SER.fromComposite( composite );
- final String edgeType = composite.readString();
- final Id targetId = ID_SER.fromComposite( composite );
- final long shard = composite.readLong();
-
- return new EdgeRowKey( sourceId, edgeType, targetId, shard );
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
index 27862d0..413c2a3 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSearcher.java
@@ -4,16 +4,16 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
-import java.util.Iterator;
import java.util.List;
-import java.util.Set;
import org.apache.usergrid.persistence.core.astyanax.ColumnParser;
+import org.apache.usergrid.persistence.core.astyanax.ColumnSearch;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.Edge;
+import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
@@ -30,23 +30,26 @@ import com.netflix.astyanax.util.RangeBuilder;
* @param <C> The column type
* @param <T> The parsed return type
*/
-public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Comparator<T> {
+public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, ColumnSearch<T>{
- protected final Optional<Edge> last;
+ protected final Optional<T> last;
protected final long maxTimestamp;
protected final ApplicationScope scope;
protected final Collection<Shard> shards;
+ protected final SearchByEdgeType.Order order;
+ protected final Comparator<T> comparator;
- protected EdgeSearcher( final ApplicationScope scope, final long maxTimestamp, final Optional<Edge> last,
- final Collection<Shard> shards ) {
+ protected EdgeSearcher( final ApplicationScope scope, final Collection<Shard> shards, final SearchByEdgeType.Order order, final Comparator<T> comparator, final long maxTimestamp, final Optional<T> last) {
Preconditions.checkArgument(shards.size() > 0 , "Cannot search with no possible shards");
this.scope = scope;
this.maxTimestamp = maxTimestamp;
- this.last = last;
+ this.order = order;
this.shards = shards;
+ this.last = last;
+ this.comparator = comparator;
}
@@ -69,20 +72,6 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Compa
- /**
- * Set the range on a search
- */
- public void setRange( final RangeBuilder builder ) {
-
- //set our start range since it was supplied to us
- if ( last.isPresent() ) {
- C sourceEdge = getStartColumn( last.get() );
-
-
- builder.setStart( sourceEdge, getSerializer() );
- }
-
- }
public boolean hasPage() {
@@ -98,6 +87,50 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Compa
}
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder, final T value ) {
+
+ C edge = createColumn( value );
+
+ rangeBuilder.setStart( edge, getSerializer() );
+
+ setRangeOptions( rangeBuilder );
+ }
+
+
+ @Override
+ public void buildRange( final RangeBuilder rangeBuilder ) {
+
+ //set our start range since it was supplied to us
+ if ( last.isPresent() ) {
+ C sourceEdge = createColumn( last.get() );
+
+
+ rangeBuilder.setStart( sourceEdge, getSerializer() );
+ }
+
+
+ setRangeOptions(rangeBuilder);
+
+
+ }
+
+ private void setRangeOptions(final RangeBuilder rangeBuilder){
+ //if we're ascending, this is opposite what cassandra sorts, so set the reversed flag
+ final boolean reversed = order == SearchByEdgeType.Order.ASCENDING;
+
+ rangeBuilder.setReversed( reversed );
+ }
+
+
+ /**
+ * Get the comparator
+ * @return
+ */
+ public Comparator<T> getComparator() {
+ return comparator;
+ }
+
/**
* Get the column's serializer
@@ -116,7 +149,7 @@ public abstract class EdgeSearcher<R, C, T> implements ColumnParser<C, T>, Compa
/**
* Set the start column to begin searching from. The last is provided
*/
- protected abstract C getStartColumn( final Edge last );
+ protected abstract C createColumn( final T last );
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSerializer.java
deleted file mode 100644
index d93f679..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeSerializer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-
-package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
-
-
-import java.nio.ByteBuffer;
-
-import org.apache.usergrid.persistence.core.astyanax.IdColDynamicCompositeSerializer;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.google.common.base.Preconditions;
-import com.netflix.astyanax.model.DynamicComposite;
-import com.netflix.astyanax.serializers.AbstractSerializer;
-import com.netflix.astyanax.serializers.LongSerializer;
-
-
-/**
- * Serializes to a source->target edge Note that we cannot set the edge type on de-serialization. Only the target
- * Id and version.
- */
-public class EdgeSerializer extends AbstractSerializer<DirectedEdge> {
-
- private static final IdColDynamicCompositeSerializer ID_COL_SERIALIZER = IdColDynamicCompositeSerializer.get();
- private static final LongSerializer LONG_SERIALIZER = LongSerializer.get();
-
-
- @Override
- public ByteBuffer toByteBuffer( final DirectedEdge edge ) {
-
- DynamicComposite composite = new DynamicComposite();
-
- composite.addComponent( edge.timestamp, LONG_SERIALIZER );
-
- ID_COL_SERIALIZER.toComposite( composite, edge.id );
-
- return composite.serialize();
- }
-
-
- @Override
- public DirectedEdge fromByteBuffer( final ByteBuffer byteBuffer ) {
- DynamicComposite composite = DynamicComposite.fromByteBuffer( byteBuffer );
-
- Preconditions.checkArgument( composite.size() == 3, "Composite should have 3 elements" );
-
-
- //return the version
- final long timestamp = composite.get( 0, LONG_SERIALIZER );
-
-
- //parse our id
- final Id id = ID_COL_SERIALIZER.fromComposite( composite, 1 );
-
-
- return new DirectedEdge( id, timestamp );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardRowKeySerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardRowKeySerializer.java
deleted file mode 100644
index 0451d68..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardRowKeySerializer.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-
-package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
-
-
-import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
-import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeType;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-
-
-public class EdgeShardRowKeySerializer implements CompositeFieldSerializer<DirectedEdgeMeta> {
-
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
- public static final EdgeShardRowKeySerializer INSTANCE = new EdgeShardRowKeySerializer();
-
-
- @Override
- public void toComposite( final CompositeBuilder builder, final DirectedEdgeMeta meta ) {
-
-
- final DirectedEdgeMeta.NodeMeta[] nodeMeta = meta.getNodes();
-
- //add the stored value
- builder.addInteger( meta.getType().getStorageValue() );
-
- final int length = nodeMeta.length;
-
- builder.addInteger( length );
-
-
- for ( DirectedEdgeMeta.NodeMeta node : nodeMeta ) {
- ID_SER.toComposite( builder, node.getId() );
- builder.addInteger( node.getNodeType().getStorageValue() );
- }
-
- final String[] edgeTypes = meta.getTypes();
-
- builder.addInteger( edgeTypes.length );
-
- for ( String type : edgeTypes ) {
- builder.addString( type );
- }
- }
-
-
- @Override
- public DirectedEdgeMeta fromComposite( final CompositeParser composite ) {
-
-
- final int storageType = composite.readInteger();
-
- final DirectedEdgeMeta.MetaType metaType = DirectedEdgeMeta.MetaType.fromStorage( storageType );
-
- final int idLength = composite.readInteger();
-
- final DirectedEdgeMeta.NodeMeta[] nodePairs = new DirectedEdgeMeta.NodeMeta[idLength];
-
-
- for ( int i = 0; i < idLength; i++ ) {
- final Id sourceId = ID_SER.fromComposite( composite );
-
- final NodeType type = NodeType.get( composite.readInteger() );
-
- nodePairs[i] = new DirectedEdgeMeta.NodeMeta( sourceId, type );
- }
-
-
- final int length = composite.readInteger();
-
- String[] types = new String[length];
-
- for ( int i = 0; i < length; i++ ) {
- types[i] = composite.readString();
- }
-
- return DirectedEdgeMeta.fromStorage( metaType, nodePairs, types );
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
index 8233d0d..3b1f37a 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
@@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeShardSerialization;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.serialize.EdgeShardRowKeySerializer;
import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import com.google.common.base.Optional;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
index b919ad7..832d9b0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardAllocationImpl.java
@@ -32,6 +32,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.graph.GraphFig;
import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
@@ -40,16 +41,15 @@ import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardA
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupCompaction;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardedEdgeSerialization;
import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
-import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import com.fasterxml.uuid.impl.UUIDUtil;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.util.TimeUUIDUtils;
/**
@@ -68,6 +68,7 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
private final NodeShardApproximation nodeShardApproximation;
private final TimeService timeService;
private final GraphFig graphFig;
+ private final ShardGroupCompaction shardGroupCompaction;
@Inject
@@ -75,13 +76,14 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
final EdgeColumnFamilies edgeColumnFamilies,
final ShardedEdgeSerialization shardedEdgeSerialization,
final NodeShardApproximation nodeShardApproximation, final TimeService timeService,
- final GraphFig graphFig ) {
+ final GraphFig graphFig, final ShardGroupCompaction shardGroupCompaction ) {
this.edgeShardSerialization = edgeShardSerialization;
this.edgeColumnFamilies = edgeColumnFamilies;
this.shardedEdgeSerialization = shardedEdgeSerialization;
this.nodeShardApproximation = nodeShardApproximation;
this.timeService = timeService;
this.graphFig = graphFig;
+ this.shardGroupCompaction = shardGroupCompaction;
}
@@ -113,7 +115,8 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
existingShards = Collections.singleton( MIN_SHARD ).iterator();
}
- return new ShardEntryGroupIterator( existingShards, graphFig.getShardMinDelta() );
+ return new ShardEntryGroupIterator( existingShards, graphFig.getShardMinDelta(), shardGroupCompaction, scope,
+ directedEdgeMeta );
}
@@ -158,18 +161,31 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
final long count = nodeShardApproximation.getCount( scope, shard, directedEdgeMeta );
+ final long shardSize = graphFig.getShardSize();
- if ( count < graphFig.getShardSize() ) {
+
+ if ( count < shardSize ) {
return false;
}
+ /**
+ * We want to allocate a new shard as close to the max value as possible. This way if we're filling up a shard rapidly, we split it near the head of the values.
+ * Further checks to this group will result in more splits, similar to creating a tree type structure and splitting each node.
+ *
+ * This means that the lower shard can be re-split later if it is still too large. We do the division to truncate
+ * to a split point < what our current max is that would be approximately be our pivot ultimately if we split from the
+ * lower bound and moved forward. Doing this will stop the current shard from expanding and avoid a point where we cannot
+ * ultimately compact to the correct shard size.
+ */
+
/**
* Allocate the shard
*/
final Iterator<MarkedEdge> edges = directedEdgeMeta
- .loadEdges( shardedEdgeSerialization, edgeColumnFamilies, scope, shardEntryGroup.getReadShards(), Long.MAX_VALUE );
+ .loadEdges( shardedEdgeSerialization, edgeColumnFamilies, scope, shardEntryGroup.getReadShards(), 0,
+ SearchByEdgeType.Order.ASCENDING );
if ( !edges.hasNext() ) {
@@ -178,14 +194,41 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
return false;
}
- //we have a next, allocate it based on the max
- MarkedEdge marked = edges.next();
+ MarkedEdge marked = null;
+
+ /**
+ * Advance to the pivot point we should use. Once it's compacted, we can split again.
+ * We either want to take the first one (unlikely) or we take our total count - the shard size.
+ * If this is a negative number, we're approaching our max count for this shard, so the first
+ * element will suffice.
+ */
+
+
+ for(long i = 1; edges.hasNext(); i++){
+ //we hit a pivot shard, set it since it could be the last one we encounter
+ if(i% shardSize == 0){
+ marked = edges.next();
+ }
+ else{
+ edges.next();
+ }
+ }
+
+
+ /**
+ * Sanity check in case our counters become severely out of sync with our edge state in cassandra.
+ */
+ if(marked == null){
+ LOG.warn( "Incorrect shard count for shard group {}", shardEntryGroup );
+ return false;
+ }
final long createTimestamp = timeService.getCurrentTime();
final Shard newShard = new Shard( marked.getTimestamp(), createTimestamp, false );
+ LOG.info( "Allocating new shard {} for edge meta {}", newShard, directedEdgeMeta );
final MutationBatch batch = this.edgeShardSerialization.writeShardMeta( scope, newShard, directedEdgeMeta );
@@ -220,22 +263,30 @@ public class NodeShardAllocationImpl implements NodeShardAllocation {
*/
private boolean isNewNode( DirectedEdgeMeta directedEdgeMeta ) {
+
+ //TODO: TN this is broken....
//The timeout is in milliseconds. Time for a time uuid is 1/10000 of a milli, so we need to get the units correct
- final long uuidDelta = graphFig.getShardCacheTimeout() * 10000;
+ final long timeoutDelta = graphFig.getShardCacheTimeout() ;
+
+ final long timeNow = timeService.getCurrentTime();
- final long timeNow = UUIDGenerator.newTimeUUID().timestamp();
+ boolean isNew = true;
for ( DirectedEdgeMeta.NodeMeta node : directedEdgeMeta.getNodes() ) {
- final long uuidTime = node.getId().getUuid().timestamp();
+ //short circuit
+ if(!isNew){
+ return false;
+ }
- final long uuidTimeDelta = uuidTime + uuidDelta;
+ final long uuidTime = TimeUUIDUtils.getTimeFromUUID( node.getId().getUuid());
- if ( uuidTimeDelta < timeNow ) {
- return true;
- }
+ final long newExpirationTimeout = uuidTime + timeoutDelta;
+
+ //our expiration is after our current time, treat it as new
+ isNew = isNew && newExpirationTimeout > timeNow;
}
- return false;
+ return isNew;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
index d444eec..36a7bc1 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/NodeShardCacheImpl.java
@@ -299,13 +299,13 @@ public class NodeShardCacheImpl implements NodeShardCache {
/**
- * Get all shards <= this one in decending order
+ * Get all shards <= this one in descending order
*/
public Iterator<ShardEntryGroup> getShards( final Long maxShard ) {
final Long firstKey = shards.floorKey( maxShard );
- return shards.headMap( firstKey, true ).descendingMap().values().iterator();
+ return Collections.unmodifiableCollection( shards.headMap( firstKey, true ).descendingMap().values()).iterator();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowSerializer.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowSerializer.java
deleted file mode 100644
index 1edaf21..0000000
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/RowSerializer.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- *
- * * Licensed to the Apache Software Foundation (ASF) under one
- * * or more contributor license agreements. See the NOTICE file
- * * distributed with this work for additional information
- * * regarding copyright ownership. The ASF licenses this file
- * * to you under the Apache License, Version 2.0 (the
- * * "License"); you may not use this file except in compliance
- * * with the License. You may obtain a copy of the License at
- * *
- * * http://www.apache.org/licenses/LICENSE-2.0
- * *
- * * Unless required by applicable law or agreed to in writing,
- * * software distributed under the License is distributed on an
- * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * * KIND, either express or implied. See the License for the
- * * specific language governing permissions and limitations
- * * under the License.
- *
- */
-
-package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl;
-
-
-import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
-import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.RowKey;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-
-
-/**
- * Class to perform serialization for row keys from edges
- */
-public class RowSerializer implements CompositeFieldSerializer<RowKey> {
-
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
-
- @Override
- public void toComposite( final CompositeBuilder builder, final RowKey key ) {
-
- //add the row id to the composite
- ID_SER.toComposite( builder, key.nodeId );
-
- builder.addString( key.edgeType );
- builder.addLong( key.shardId );
- }
-
-
- @Override
- public RowKey fromComposite( final CompositeParser composite ) {
-
- final Id id = ID_SER.fromComposite( composite );
- final String edgeType = composite.readString();
- final long shard = composite.readLong();
-
-
- return new RowKey( id, edgeType, shard );
- }
-}
[2/5] Updated OrderedMerge to use a faster implementation at runtime.
After initialization,
it's an O(1) emit operation as long as our produces are fast enough.
Posted by to...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
index d0a0525..d39e433 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
@@ -112,7 +112,7 @@ public class GraphManagerLoadTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), null ) );
+ return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null) );
}
};
@@ -139,7 +139,7 @@ public class GraphManagerLoadTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), null ) );
+ return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
}
};
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index 59bf014..c01d157 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -21,27 +21,31 @@
package org.apache.usergrid.persistence.graph;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import java.util.UUID;
import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import javax.annotation.Nullable;
import org.junit.After;
import org.junit.Before;
import org.junit.ClassRule;
-import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.Marker;
-
-import org.apache.commons.lang.time.StopWatch;
import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
import org.apache.usergrid.persistence.core.migration.MigrationException;
@@ -51,29 +55,33 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardAllocation;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardCache;
-import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
import org.apache.usergrid.persistence.model.entity.Id;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Slf4jReporter;
-import com.google.common.base.Optional;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.netflix.config.ConfigurationManager;
import rx.Observable;
+import rx.functions.Action1;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createEdge;
import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
-@Ignore("A stress test, not part of functional testing")
+//@Ignore( "A stress test, not part of functional testing" )
public class GraphManagerShardConsistencyIT {
private static final Logger log = LoggerFactory.getLogger( GraphManagerShardConsistencyIT.class );
@@ -85,12 +93,9 @@ public class GraphManagerShardConsistencyIT {
private static final Meter writeMeter = registry.meter( "writeThroughput" );
- private static final Slf4jReporter reporter = Slf4jReporter.forRegistry( registry )
- .outputTo( log )
- .convertRatesTo( TimeUnit.SECONDS )
- .convertDurationsTo( TimeUnit.MILLISECONDS )
- .build();
-
+ private static final Slf4jReporter reporter =
+ Slf4jReporter.forRegistry( registry ).outputTo( log ).convertRatesTo( TimeUnit.SECONDS )
+ .convertDurationsTo( TimeUnit.MILLISECONDS ).build();
protected ApplicationScope scope;
@@ -102,6 +107,7 @@ public class GraphManagerShardConsistencyIT {
protected Object originalShardDelta;
+
@Before
public void setupOrg() {
@@ -110,82 +116,52 @@ public class GraphManagerShardConsistencyIT {
originalShardTimeout = ConfigurationManager.getConfigInstance().getProperty( GraphFig.SHARD_CACHE_TIMEOUT );
- originalShardDelta = ConfigurationManager.getConfigInstance().getProperty( GraphFig.SHARD_MIN_DELTA );
+ originalShardDelta = ConfigurationManager.getConfigInstance().getProperty( GraphFig.SHARD_MIN_DELTA );
- ConfigurationManager.getConfigInstance().setProperty( GraphFig.SHARD_SIZE, 10000 );
+ ConfigurationManager.getConfigInstance().setProperty( GraphFig.SHARD_SIZE, 1000 );
- final long cacheTimeout = 10000;
- //set our cache timeout to 10 seconds
+ final long cacheTimeout = 5000;
+ //set our cache timeout to the above value
ConfigurationManager.getConfigInstance().setProperty( GraphFig.SHARD_CACHE_TIMEOUT, cacheTimeout );
- final long minDelta = ( long ) (cacheTimeout * 2.5);
+ final long minDelta = ( long ) ( cacheTimeout * 2.5 );
ConfigurationManager.getConfigInstance().setProperty( GraphFig.SHARD_MIN_DELTA, minDelta );
-
-
//get the system property of the UUID to use. If one is not set, use the defualt
String uuidString = System.getProperty( "org.id", "80a42760-b699-11e3-a5e2-0800200c9a66" );
scope = new ApplicationScopeImpl( createId( UUID.fromString( uuidString ), "test" ) );
-
reporter.start( 10, TimeUnit.SECONDS );
}
@After
- public void tearDown(){
+ public void tearDown() {
reporter.stop();
reporter.report();
}
-// @Test
-// public void writeThousandsSingleSource() throws InterruptedException, ExecutionException {
-// EdgeGenerator generator = new EdgeGenerator() {
-//
-// private Id sourceId = createId( "source" );
-//
-//
-// @Override
-// public Edge newEdge() {
-// Edge edge = createEdge( sourceId, "test", createId( "target" ) );
-//
-//
-// return edge;
-// }
-//
-//
-// @Override
-// public Observable<Edge> doSearch( final GraphManager manager ) {
-// return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), null ) );
-// }
-// };
-//
-//
-//
-// doTest( generator );
-// }
-
-
@Test
- public void writeThousandsSingleTarget() throws InterruptedException, ExecutionException, MigrationException {
+ public void writeThousandsSingleSource()
+ throws InterruptedException, ExecutionException, MigrationException, UnsupportedEncodingException {
- final Id sourceId = createId("source");
+ final Id sourceId = createId( "source" );
final String edgeType = "test";
- EdgeGenerator generator = new EdgeGenerator() {
+ final EdgeGenerator generator = new EdgeGenerator() {
@Override
public Edge newEdge() {
- Edge edge = createEdge( sourceId, edgeType, createId( "target" ) );
+ Edge edge = createEdge( sourceId, edgeType, createId( "target" ) );
return edge;
@@ -194,59 +170,84 @@ public class GraphManagerShardConsistencyIT {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), null ) );
+ return manager.loadEdgesFromSource(
+ new SimpleSearchByEdgeType( sourceId, edgeType, Long.MAX_VALUE,
+ SearchByEdgeType.Order.DESCENDING, null ) );
}
};
- final int numInjectors = 2;
+// final int numInjectors = 2;
+ final int numInjectors = 1;
/**
* create 3 injectors. This way all the caches are independent of one another. This is the same as
* multiple nodes
*/
- final List<Injector> injectors = createInjectors(numInjectors);
+ final List<Injector> injectors = createInjectors( numInjectors );
final GraphFig graphFig = getInstance( injectors, GraphFig.class );
- final long shardSize = graphFig.getShardSize();
+ final long shardSize = graphFig.getShardSize();
- //we don't want to starve the cass runtime since it will be on the same box. Only take 50% of processing power for writes
- final int numProcessors = Runtime.getRuntime().availableProcessors() /2 ;
+ //we don't want to starve the cass runtime since it will be on the same box. Only take 50% of processing
+ // power for writes
+ final int numProcessors = Runtime.getRuntime().availableProcessors() / 2;
- final int numWorkers = numProcessors/numInjectors;
+ final int numWorkersPerInjector = numProcessors / numInjectors;
/**
* Do 4x shard size so we should have approximately 4 shards
*/
- final long numberOfEdges = shardSize * 4;
+ final long numberOfEdges = shardSize * 4;
- final long countPerWorker = numberOfEdges/numWorkers;
+ final long workerWriteLimit = numberOfEdges / numWorkersPerInjector;
- final long writeLimit = countPerWorker;
+ final long expectedShardCount = numberOfEdges/shardSize;
+
+
+ final ListeningExecutorService
+ executor = MoreExecutors.listeningDecorator( Executors.newFixedThreadPool( numWorkersPerInjector ) );
+
+
+ final AtomicLong writeCounter = new AtomicLong();
+
//min stop time the min delta + 1 cache cycle timeout
final long minExecutionTime = graphFig.getShardMinDelta() + graphFig.getShardCacheTimeout();
-
+ log.info( "Writing {} edges per worker on {} workers in {} injectors", workerWriteLimit, numWorkersPerInjector,
+ numInjectors );
final List<Future<Boolean>> futures = new ArrayList<>();
- for(Injector injector: injectors) {
+
+
+ for ( Injector injector : injectors ) {
final GraphManagerFactory gmf = injector.getInstance( GraphManagerFactory.class );
- futures.addAll( doTest( gmf, generator, numWorkers, writeLimit, minExecutionTime ) );
+
+ for ( int i = 0; i < numWorkersPerInjector; i++ ) {
+ Future<Boolean> future = executor
+ .submit( new Worker( gmf, generator, workerWriteLimit, minExecutionTime, writeCounter ) );
+
+ futures.add( future );
+ }
+
}
- for(Future<Boolean> future: futures){
+ /**
+ * Wait for all writes to complete
+ */
+ for ( Future<Boolean> future : futures ) {
future.get();
}
@@ -255,53 +256,134 @@ public class GraphManagerShardConsistencyIT {
final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( sourceId, edgeType );
- int count = 0;
+ //now submit the readers.
+ final GraphManagerFactory gmf = getInstance( injectors, GraphManagerFactory.class );
- while(true) {
- //reset our count. Ultimately we'll have 4 groups once our compaction completes
- count = 0;
+ final long writeCount = writeCounter.get();
+ final Meter readMeter = registry.meter( "readThroughput" );
+
+ /**
+ * Start reading continuously while we migrate data to ensure our view is always correct
+ */
+ final ListenableFuture<Long> future = executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
+
+ final List<Throwable> failures = new ArrayList<>( );
+
+
+
+
+ //add the future
+ Futures.addCallback( future, new FutureCallback<Long>() {
+
+ @Override
+ public void onSuccess( @Nullable final Long result ) {
+ log.info( "Successfully ran the read, re-running" );
+ executor.submit( new ReadWorker( gmf, generator, writeCount, readMeter ) );
+ }
+
+
+ @Override
+ public void onFailure( final Throwable t ) {
+ failures.add( t );
+ log.error( "Failed test!", t );
+ }
+ } );
+
+
+
+ int compactedCount;
+
+
+
+
+
+ //now start our readers
+
+ while ( true ) {
+
+ if(!failures.isEmpty()){
+
+ StringBuilder builder = new StringBuilder( );
+
+ builder.append("Read runner failed!\n");
+
+ for(Throwable t: failures){
+ builder.append( "Exception is: " );
+ ByteArrayOutputStream output = new ByteArrayOutputStream( );
+
+ t.printStackTrace( new PrintWriter( output ) );
+
+ builder.append( output.toString( "UTF-8" ));
+ builder.append( "\n\n" );
+
+ }
+
+
+
+ fail(builder.toString());
+ }
+
+ //reset our count. Ultimately we'll have 4 groups once our compaction completes
+ compactedCount = 0;
//we have to get it from the cache, because this will trigger the compaction process
final Iterator<ShardEntryGroup> groups = cache.getReadShardGroup( scope, Long.MAX_VALUE, directedEdgeMeta );
+ final Set<ShardEntryGroup> shardEntryGroups = new HashSet<>();
+
+ while ( groups.hasNext() ) {
- while(groups.hasNext()){
final ShardEntryGroup group = groups.next();
+ shardEntryGroups.add( group );
log.info( "Compaction pending status for group {} is {}", group, group.isCompactionPending() );
- count++;
-
+ if ( !group.isCompactionPending() ) {
+ compactedCount++;
+ }
}
+
//we're done
- if(count == 4){
+ if ( compactedCount >= expectedShardCount ) {
+ log.info( "All compactions complete, sleeping" );
+
+// final Object mutex = new Object();
+//
+// synchronized ( mutex ){
+//
+// mutex.wait();
+// }
+
break;
+
}
- Thread.sleep(5000);
- }
+ Thread.sleep( 2000 );
+ }
+
+ executor.shutdownNow();
}
- private <T> T getInstance(final List<Injector> injectors, Class<T> clazz ){
+
+ private <T> T getInstance( final List<Injector> injectors, Class<T> clazz ) {
return injectors.get( 0 ).getInstance( clazz );
}
/**
* Create new Guice injector environments and return them
- * @param count
*/
private List<Injector> createInjectors( int count ) throws MigrationException {
- final List<Injector> injectors = new ArrayList<>(count);
+ final List<Injector> injectors = new ArrayList<>( count );
- for(int i = 0; i < count; i++){
+ for ( int i = 0; i < count; i++ ) {
final Injector injector = Guice.createInjector( new TestGraphModule() );
injectors.add( injector );
}
@@ -312,42 +394,27 @@ public class GraphManagerShardConsistencyIT {
migrationManager.migrate();
return injectors;
-
-
}
- /**
- * Execute the test with the generator
- */
- private List<Future<Boolean>> doTest(final GraphManagerFactory factory, final EdgeGenerator generator, final int numWorkers, final long writeCount, final long minExecutionTime ) throws InterruptedException, ExecutionException {
-
- ExecutorService executor = Executors.newFixedThreadPool( numWorkers );
-
- List<Future<Boolean>> futures = new ArrayList<>( numWorkers );
- for ( int i = 0; i < numWorkers; i++ ) {
- Future<Boolean> future = executor.submit( new Worker(factory, generator, writeCount, minExecutionTime ) );
-
- futures.add( future );
- }
- return futures;
- }
-
private class Worker implements Callable<Boolean> {
private final GraphManagerFactory factory;
private final EdgeGenerator generator;
private final long writeLimit;
private final long minExecutionTime;
+ private final AtomicLong writeCounter;
- private Worker( final GraphManagerFactory factory, final EdgeGenerator generator, final long writeLimit, final long minExecutionTime ) {
+ private Worker( final GraphManagerFactory factory, final EdgeGenerator generator, final long writeLimit,
+ final long minExecutionTime, final AtomicLong writeCounter ) {
this.factory = factory;
this.generator = generator;
this.writeLimit = writeLimit;
this.minExecutionTime = minExecutionTime;
+ this.writeCounter = writeCounter;
}
@@ -356,11 +423,10 @@ public class GraphManagerShardConsistencyIT {
GraphManager manager = factory.createEdgeManager( scope );
-
final long startTime = System.currentTimeMillis();
- for ( long i = 0; i < writeLimit || System.currentTimeMillis() - startTime < minExecutionTime ; i++ ) {
+ for ( long i = 0; i < writeLimit || System.currentTimeMillis() - startTime < minExecutionTime; i++ ) {
Edge edge = generator.newEdge();
@@ -372,6 +438,10 @@ public class GraphManagerShardConsistencyIT {
writeMeter.mark();
+ writeCounter.incrementAndGet();
+
+
+
if ( i % 1000 == 0 ) {
log.info( " Wrote: " + i );
}
@@ -383,6 +453,94 @@ public class GraphManagerShardConsistencyIT {
}
+ private class ReadWorker implements Callable<Long> {
+ private final GraphManagerFactory factory;
+ private final EdgeGenerator generator;
+ private final long writeCount;
+ private final Meter readMeter;
+
+ private ReadWorker( final GraphManagerFactory factory, final EdgeGenerator generator, final long writeCount,
+ final Meter readMeter) {
+ this.factory = factory;
+ this.generator = generator;
+ this.writeCount = writeCount;
+ this.readMeter = readMeter;
+ }
+
+
+ @Override
+ public Long call() throws Exception {
+
+
+
+
+ GraphManager gm = factory.createEdgeManager( scope );
+
+
+
+ while(true) {
+
+// final long[] count = {0};
+// final long[] duplicate = {0};
+// final HashSet<Edge > seen = new HashSet<>((int)writeCount);
+
+
+ //do a read to eventually trigger our group compaction. Take 2 pages of columns
+ final long returnedEdgeCount = generator.doSearch( gm )
+
+ .doOnNext( new Action1<Edge>() {
+
+
+
+// private Edge last;
+
+
+ @Override
+ public void call( final Edge edge ) {
+ readMeter.mark();
+
+// count[0]++;
+//
+// /**
+// * Added this check as part of the read
+// */
+// if ( last != null && last.equals(edge) ) {
+// fail( String.format( "Expected edges to be in order, however last was %s and current is %s",
+// last, edge ) );
+// }
+//
+// last = edge;
+//
+// if( seen.contains( edge ) ){
+// fail( String.format("Returned an edge that was already seen! Edge was %s, last edge was %s", edge, last) );
+// duplicate[0]++;
+// }
+//
+// seen.add( edge );
+
+ }
+ } )
+
+ .longCount().toBlocking().last();
+
+
+// if(returnedEdgeCount != count[0]-duplicate[0]){
+// log.warn( "Missing entries from the initial put" );
+// }
+
+ log.info( "Completed reading {} edges", returnedEdgeCount );
+
+ if(writeCount != returnedEdgeCount){
+ log.warn( "Unexpected edge count returned!!! Expected {} but was {}", writeCount, returnedEdgeCount );
+ }
+
+ assertEquals( "Expected to read same edge count", writeCount, returnedEdgeCount );
+ }
+
+ }
+ }
+
+
private interface EdgeGenerator {
/**
@@ -392,13 +550,9 @@ public class GraphManagerShardConsistencyIT {
/**
* Perform the search returning an observable edge
- * @param manager
- * @return
*/
public Observable<Edge> doSearch( final GraphManager manager );
}
-
-
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
index cf6dda8..e4b67c2 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
@@ -118,7 +118,7 @@ public class GraphManagerStressTest {
for ( Id sourceId : sourceIds ) {
final Iterable<Edge> edges = manager.loadEdgesFromSource(
- new SimpleSearchByEdgeType( sourceId, "test", timestamp, null ) )
+ new SimpleSearchByEdgeType( sourceId, "test", timestamp, SearchByEdgeType.Order.DESCENDING, null ) )
.toBlocking().toIterable();
for ( Edge edge : edges ) {
@@ -192,7 +192,7 @@ public class GraphManagerStressTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), null ) );
+ return manager.loadEdgesFromSource( new SimpleSearchByEdgeType( sourceId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
}
};
@@ -220,7 +220,7 @@ public class GraphManagerStressTest {
@Override
public Observable<Edge> doSearch( final GraphManager manager ) {
- return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), null ) );
+ return manager.loadEdgesToTarget( new SimpleSearchByEdgeType( targetId, "test", System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
}
};
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
index 4b62ad1..e8af91c 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
@@ -34,6 +34,7 @@ import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
import org.apache.usergrid.persistence.core.cassandra.ITRunner;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.MarkedEdge;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdge;
import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
@@ -131,7 +132,7 @@ public class EdgeDeleteRepairTest {
Iterator<MarkedEdge> itr = storageEdgeSerialization.getEdgeVersions( scope,
- new SimpleSearchByEdge( sourceId, edgeType, targetId, System.currentTimeMillis(), null ) );
+ new SimpleSearchByEdge( sourceId, edgeType, targetId, System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
assertEquals( edge2, itr.next() );
assertEquals( edge1, itr.next() );
@@ -142,7 +143,7 @@ public class EdgeDeleteRepairTest {
assertEquals( edge1, deleted );
itr = storageEdgeSerialization.getEdgeVersions( scope,
- new SimpleSearchByEdge( sourceId, edgeType, targetId, System.currentTimeMillis(), null ) );
+ new SimpleSearchByEdge( sourceId, edgeType, targetId, System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING, null ) );
assertEquals( edge2, itr.next() );
assertFalse( itr.hasNext() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
index 878674a..b8be5d2 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/NodeShardAllocationTest.java
@@ -19,10 +19,12 @@
package org.apache.usergrid.persistence.graph.serialization.impl.shard;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
+import java.util.List;
import org.junit.Before;
import org.junit.Test;
@@ -87,6 +89,8 @@ public class NodeShardAllocationTest {
@Test
public void minTime() {
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
@@ -101,7 +105,7 @@ public class NodeShardAllocationTest {
NodeShardAllocation approximation =
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
- nodeShardCounterSerialization, timeService, graphFig );
+ nodeShardCounterSerialization, timeService, graphFig, shardGroupCompaction );
final long timeservicetime = System.currentTimeMillis();
@@ -116,9 +120,10 @@ public class NodeShardAllocationTest {
}
-
@Test
public void existingFutureShardSameTime() {
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
@@ -131,10 +136,9 @@ public class NodeShardAllocationTest {
final TimeService timeService = mock( TimeService.class );
-
NodeShardAllocation approximation =
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
- nodeShardCounterSerialization, timeService, graphFig );
+ nodeShardCounterSerialization, timeService, graphFig, shardGroupCompaction );
final Id nodeId = createId( "test" );
final String type = "type";
@@ -145,7 +149,7 @@ public class NodeShardAllocationTest {
when( timeService.getCurrentTime() ).thenReturn( timeservicetime );
- final Shard firstShard = new Shard(0l, 0l, true);
+ final Shard firstShard = new Shard( 0l, 0l, true );
final Shard futureShard = new Shard( 10000l, timeservicetime, false );
final ShardEntryGroup shardEntryGroup = new ShardEntryGroup( 1000l );
@@ -162,6 +166,8 @@ public class NodeShardAllocationTest {
@Test
public void lowCountFutureShard() {
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
@@ -175,7 +181,7 @@ public class NodeShardAllocationTest {
NodeShardAllocation approximation =
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
- nodeShardApproximation, timeService, graphFig );
+ nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
final Id nodeId = createId( "test" );
final String type = "type";
@@ -207,7 +213,9 @@ public class NodeShardAllocationTest {
@Test
- public void equalCountFutureShard() {
+ public void overAllocatedShard() {
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
@@ -221,7 +229,7 @@ public class NodeShardAllocationTest {
NodeShardAllocation approximation =
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
- nodeShardApproximation, timeService, graphFig );
+ nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
final Id nodeId = createId( "test" );
final String type = "type";
@@ -239,11 +247,58 @@ public class NodeShardAllocationTest {
final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType( nodeId, type, subType );
- final long shardCount = graphFig.getShardSize();
+
+ /**
+ * Allocate 2.5x what this shard should have. We should ultimately have a split at 2x
+ */
+ final long shardCount = ( long ) (graphFig.getShardSize() * 2.5);
+
//return a shard size equal to our max
when( nodeShardApproximation.getCount( scope, futureShard, targetEdgeMeta ) ).thenReturn( shardCount );
+
+ //this is how many we should be iterating and should set the value of the last shard we keep
+ final int numToIterate = ( int ) (graphFig.getShardSize() *2);
+
+
+ /**
+ * Just use 2 edges. It means that we won't generate a boatload of data and kill our test. We just want
+ * to check that the one we want to return is correct
+ */
+ SimpleMarkedEdge skipped = new SimpleMarkedEdge( nodeId, type, createId( subType ), 10000, false );
+ SimpleMarkedEdge keep = new SimpleMarkedEdge( nodeId, type, createId( subType ), 20000, false );
+
+ //allocate some extra to ensure we seek the right value
+ List<MarkedEdge> edges = new ArrayList( numToIterate + 100 );
+
+ int i = 0;
+
+ for (; i < numToIterate - 1; i++ ) {
+ edges.add( skipped );
+ }
+
+ //add our keep edge
+ edges.add( keep );
+ i++;
+
+ for ( ; i < shardCount; i++ ) {
+
+ edges.add( skipped );
+ }
+
+
+ final Iterator<MarkedEdge> edgeIterator = edges.iterator();
+
+ //mock up returning the value
+ when( shardedEdgeSerialization
+ .getEdgesFromSourceByTargetType( same( edgeColumnFamilies ), same( scope ), any( SearchByIdType.class ),
+ any( Collection.class ) ) ).thenReturn( edgeIterator );
+
+
+ /**
+ * Mock up the write shard meta data
+ */
ArgumentCaptor<Shard> shardValue = ArgumentCaptor.forClass( Shard.class );
@@ -252,10 +307,89 @@ public class NodeShardAllocationTest {
.thenReturn( mock( MutationBatch.class ) );
+ final boolean result = approximation.auditShard( scope, shardEntryGroup, targetEdgeMeta );
+
+ assertTrue( "Shard was split correctly", result );
+
+ final long savedTimestamp = shardValue.getValue().getCreatedTime();
+
+
+ assertEquals( "Expected time service time", timeservicetime, savedTimestamp );
+
+
+ //now check our max value was set. Since our shard is significantly over allocated, we should be iterating
+ //through elements to move the pivot down to a more manageable size
+
+ final long savedShardPivot = shardValue.getValue().getShardIndex();
+
+ assertEquals( "Expected max value to be the same", keep.getTimestamp(), savedShardPivot );
+ }
+
+
+ @Test
+ public void equalCountFutureShard() {
+
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
+ final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
+
+ final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
+
+ final ShardedEdgeSerialization shardedEdgeSerialization = mock( ShardedEdgeSerialization.class );
+
+ final NodeShardApproximation nodeShardApproximation = mock( NodeShardApproximation.class );
+
+
+ final TimeService timeService = mock( TimeService.class );
+
+ NodeShardAllocation approximation =
+ new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
+ nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
+
+ final Id nodeId = createId( "test" );
+ final String type = "type";
+ final String subType = "subType";
+
+
+ final long timeservicetime = System.currentTimeMillis();
+
+ when( timeService.getCurrentTime() ).thenReturn( timeservicetime );
+
+ final Shard futureShard = new Shard( 0l, 0l, true );
+
+ final ShardEntryGroup shardEntryGroup = new ShardEntryGroup( 1000l );
+ shardEntryGroup.addShard( futureShard );
+
+ final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType( nodeId, type, subType );
+
+ final long shardCount = graphFig.getShardSize();
+
+
+ final SimpleMarkedEdge skippedEdge = new SimpleMarkedEdge( nodeId, type, createId( "subType" ), 10000l, false );
final SimpleMarkedEdge returnedEdge =
new SimpleMarkedEdge( nodeId, type, createId( "subType" ), 10005l, false );
- final Iterator<MarkedEdge> edgeIterator = Collections.singleton( ( MarkedEdge ) returnedEdge ).iterator();
+ List<MarkedEdge> iteratedEdges = new ArrayList<>( ( int ) shardCount );
+
+ for(long i = 0; i < shardCount-1; i ++){
+ iteratedEdges.add( skippedEdge);
+ }
+
+ iteratedEdges.add( returnedEdge );
+
+ //return a shard size equal to our max
+ when( nodeShardApproximation.getCount( scope, futureShard, targetEdgeMeta ) ).thenReturn( shardCount );
+
+ ArgumentCaptor<Shard> shardValue = ArgumentCaptor.forClass( Shard.class );
+
+
+ //mock up our mutation
+ when( edgeShardSerialization.writeShardMeta( same( scope ), shardValue.capture(), same( targetEdgeMeta ) ) )
+ .thenReturn( mock( MutationBatch.class ) );
+
+
+
+ final Iterator<MarkedEdge> edgeIterator = iteratedEdges.iterator();
//mock up returning the value
when( shardedEdgeSerialization
@@ -285,7 +419,76 @@ public class NodeShardAllocationTest {
@Test
+ public void invalidCountNoShards() {
+
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
+ final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
+
+ final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
+
+ final ShardedEdgeSerialization shardedEdgeSerialization = mock( ShardedEdgeSerialization.class );
+
+ final NodeShardApproximation nodeShardApproximation = mock( NodeShardApproximation.class );
+
+
+ final TimeService timeService = mock( TimeService.class );
+
+ NodeShardAllocation approximation =
+ new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
+ nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
+
+ final Id nodeId = createId( "test" );
+ final String type = "type";
+ final String subType = "subType";
+
+
+ final long timeservicetime = System.currentTimeMillis();
+
+ when( timeService.getCurrentTime() ).thenReturn( timeservicetime );
+
+ final Shard futureShard = new Shard( 0l, 0l, true );
+
+ final ShardEntryGroup shardEntryGroup = new ShardEntryGroup( 1000l );
+ shardEntryGroup.addShard( futureShard );
+
+ final DirectedEdgeMeta targetEdgeMeta = DirectedEdgeMeta.fromSourceNodeTargetType( nodeId, type, subType );
+
+ final long shardCount = graphFig.getShardSize();
+
+ //return a shard size equal to our max
+ when( nodeShardApproximation.getCount( scope, futureShard, targetEdgeMeta ) ).thenReturn( shardCount );
+
+ ArgumentCaptor<Shard> shardValue = ArgumentCaptor.forClass( Shard.class );
+
+
+ //mock up our mutation
+ when( edgeShardSerialization.writeShardMeta( same( scope ), shardValue.capture(), same( targetEdgeMeta ) ) )
+ .thenReturn( mock( MutationBatch.class ) );
+
+
+ final SimpleMarkedEdge returnedEdge =
+ new SimpleMarkedEdge( nodeId, type, createId( "subType" ), 10005l, false );
+
+ final Iterator<MarkedEdge> edgeIterator = Collections.singleton( ( MarkedEdge ) returnedEdge ).iterator();
+
+ //mock up returning the value
+ when( shardedEdgeSerialization
+ .getEdgesFromSourceByTargetType( same( edgeColumnFamilies ), same( scope ), any( SearchByIdType.class ),
+ any( Collection.class ) ) ).thenReturn( edgeIterator );
+
+
+ final boolean result = approximation.auditShard( scope, shardEntryGroup, targetEdgeMeta );
+
+ assertFalse( "Shard should not be allocated", result );
+ }
+
+
+ @Test
public void futureCountShardCleanup() {
+
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
@@ -300,7 +503,7 @@ public class NodeShardAllocationTest {
NodeShardAllocation approximation =
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
- nodeShardApproximation, timeService, graphFig );
+ nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
final Id nodeId = createId( "test" );
final String type = "type";
@@ -310,7 +513,7 @@ public class NodeShardAllocationTest {
/**
* Use the time service to generate timestamps
*/
- final long timeservicetime = 10000;
+ final long timeservicetime = System.currentTimeMillis() + 60000;
when( timeService.getCurrentTime() ).thenReturn( timeservicetime );
@@ -382,21 +585,16 @@ public class NodeShardAllocationTest {
Collection<Shard> writeShards = shardEntryGroup.getWriteShards( minTime + minDelta );
- assertEquals( "Shard size as expected", 4, writeShards.size() );
+ assertEquals( "Shard size as expected", 1, writeShards.size() );
- assertTrue( writeShards.contains( futureShard1 ) );
- assertTrue( writeShards.contains( futureShard2 ) );
- assertTrue( writeShards.contains( futureShard3 ) );
assertTrue( writeShards.contains( compactedShard ) );
Collection<Shard> readShards = shardEntryGroup.getReadShards();
- assertEquals( "Shard size as expected", 4, readShards.size() );
+ assertEquals( "Shard size as expected", 2, readShards.size() );
assertTrue( readShards.contains( futureShard1 ) );
- assertTrue( readShards.contains( futureShard2 ) );
- assertTrue( readShards.contains( futureShard3 ) );
assertTrue( readShards.contains( compactedShard ) );
@@ -423,6 +621,9 @@ public class NodeShardAllocationTest {
@Test
public void noShardsReturns() throws ConnectionException {
+
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
@@ -434,13 +635,15 @@ public class NodeShardAllocationTest {
final TimeService timeService = mock( TimeService.class );
- when( timeService.getCurrentTime() ).thenReturn( 10000l );
+ final long returnTime = System.currentTimeMillis()+graphFig.getShardCacheTimeout()*2 ;
+
+ when( timeService.getCurrentTime() ).thenReturn( returnTime );
final MutationBatch batch = mock( MutationBatch.class );
NodeShardAllocation approximation =
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
- nodeShardApproximation, timeService, graphFig );
+ nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
final Id nodeId = createId( "test" );
final String type = "type";
@@ -499,6 +702,8 @@ public class NodeShardAllocationTest {
@Test
public void invalidConfiguration() {
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
final GraphFig graphFig = mock( GraphFig.class );
final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
@@ -531,7 +736,7 @@ public class NodeShardAllocationTest {
NodeShardAllocation approximation =
new NodeShardAllocationImpl( edgeShardSerialization, edgeColumnFamilies, shardedEdgeSerialization,
- nodeShardApproximation, timeService, graphFig );
+ nodeShardApproximation, timeService, graphFig, shardGroupCompaction );
/**
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
index 6096e58..ee4b94a 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardEntryGroupTest.java
@@ -264,13 +264,11 @@ public class ShardEntryGroupTest {
Collection<Shard> readShards = shardEntryGroup.getReadShards();
- assertEquals("Shard size correct", 3, readShards.size());
+ assertEquals("Shard size correct", 2, readShards.size());
- assertTrue("First shard present", readShards.contains( firstShard ) );
+ assertTrue("First shard present", readShards.contains( secondShard ) );
- assertTrue("Second shard present", readShards.contains( firstShard ) );
-
- assertTrue("Third shard present", readShards.contains( firstShard ) );
+ assertTrue("Second shard present", readShards.contains( compactedShard1 ) );
}
@@ -308,23 +306,15 @@ public class ShardEntryGroupTest {
Collection<Shard> writeShards = shardEntryGroup.getWriteShards( firstShard.getCreatedTime() + delta );
- assertEquals("Shard size correct", 3, writeShards.size());
-
- assertTrue("First shard present", writeShards.contains( firstShard ) );
-
- assertTrue("Second shard present", writeShards.contains( secondShard ) );
+ assertEquals("Shard size correct", 1, writeShards.size());
- assertTrue("Third shard present", writeShards.contains( compactedShard ) );
+ assertTrue("Root shard present", writeShards.contains( compactedShard ) );
writeShards = shardEntryGroup.getWriteShards(secondShard.getCreatedTime()+delta);
- assertEquals("Shard size correct", 3, writeShards.size());
-
- assertTrue("First shard present", writeShards.contains( firstShard ) );
-
- assertTrue("Second shard present", writeShards.contains( secondShard ) );
+ assertEquals("Shard size correct", 1, writeShards.size());
assertTrue("Third shard present", writeShards.contains( compactedShard ) );
@@ -334,13 +324,11 @@ public class ShardEntryGroupTest {
*/
writeShards = shardEntryGroup.getWriteShards(secondShard.getCreatedTime() +1 + delta);
- assertEquals("Shard size correct", 3, writeShards.size());
+ assertEquals("Shard size correct", 1, writeShards.size());
- assertTrue("First shard present", writeShards.contains( firstShard ) );
- assertTrue("Second shard present", writeShards.contains( secondShard ) );
+ assertTrue("Second shard present", writeShards.contains( compactedShard ) );
- assertTrue("Third shard present", writeShards.contains( compactedShard ) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
new file mode 100644
index 0000000..1513e85
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/ShardGroupCompactionTest.java
@@ -0,0 +1,226 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard;
+
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Set;
+
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.Matchers;
+
+import org.apache.usergrid.persistence.core.consistency.TimeService;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.graph.GraphFig;
+import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.ShardGroupCompactionImpl;
+
+import com.netflix.astyanax.Keyspace;
+
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+
+public class ShardGroupCompactionTest {
+
+ protected GraphFig graphFig;
+ protected ApplicationScope scope;
+
+
+ @Before
+ public void setup() {
+ graphFig = mock( GraphFig.class );
+
+ when( graphFig.getShardAuditWorkerCount() ).thenReturn( 10 );
+
+ when( graphFig.getShardAuditWorkerQueueSize() ).thenReturn( 1000 );
+
+ this.scope = new ApplicationScopeImpl( createId( "application" ) );
+ }
+
+
+ @Test
+ public void shouldNotCompact() {
+
+ final TimeService timeService = mock( TimeService.class );
+
+ final NodeShardAllocation nodeShardAllocation = mock( NodeShardAllocation.class );
+
+ final ShardedEdgeSerialization shardedEdgeSerialization = mock( ShardedEdgeSerialization.class );
+
+ final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
+
+ final Keyspace keyspace = mock( Keyspace.class );
+
+ final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
+
+ final long delta = 10000;
+
+ final long createTime = 20000;
+
+ //we shouldn't be able to compact, should throw an exception
+ final long timeNow = createTime + delta - 1;
+
+ ShardEntryGroup group = new ShardEntryGroup( delta );
+ group.addShard( new Shard( 2000, createTime, false ) );
+ group.addShard( new Shard( 1000, 5000, true ) );
+
+
+ when( timeService.getCurrentTime() ).thenReturn( timeNow );
+
+ ShardGroupCompactionImpl compaction =
+ new ShardGroupCompactionImpl( timeService, graphFig, nodeShardAllocation, shardedEdgeSerialization,
+ edgeColumnFamilies, keyspace, edgeShardSerialization );
+
+
+ DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId("source"), "test" );
+
+ try {
+ compaction.compact( this.scope, directedEdgeMeta, group );
+ fail( "I should not reach this point" );
+ }catch(Throwable t){
+ assertEquals("Correct error message returned", "Compaction cannot be run yet. Ignoring compaction.", t.getMessage());
+ }
+
+ }
+
+
+// /**
+// * Tests that when we copy edges, we do not actually run the compaction, we can only run it after we get nothing
+// * and the timeout has elapsed
+// */
+// @Test
+// public void shouldOnlyCopy() {
+//
+// final TimeService timeService = mock( TimeService.class );
+//
+// final NodeShardAllocation nodeShardAllocation = mock( NodeShardAllocation.class );
+//
+// final ShardedEdgeSerialization shardedEdgeSerialization = mock( ShardedEdgeSerialization.class );
+//
+// final EdgeColumnFamilies edgeColumnFamilies = mock( EdgeColumnFamilies.class );
+//
+// final Keyspace keyspace = mock( Keyspace.class );
+//
+// final EdgeShardSerialization edgeShardSerialization = mock( EdgeShardSerialization.class );
+//
+// final long delta = 10000;
+//
+// final long createTime = 20000;
+//
+// //we shouldn't be able to compact, should throw an exception
+// final long timeNow = createTime + delta ;
+//
+//
+// final Shard targetShard = new Shard( 2000, createTime, false ) ;
+// final Shard sourceShard = new Shard( 1000, 5000, true );
+// ShardEntryGroup group = new ShardEntryGroup( delta );
+// group.addShard( targetShard );
+// group.addShard( sourceShard );
+//
+//
+// when( timeService.getCurrentTime() ).thenReturn( timeNow );
+//
+// ShardGroupCompaction compaction =
+// new ShardGroupCompactionImpl( timeService, graphFig, nodeShardAllocation, shardedEdgeSerialization,
+// edgeColumnFamilies, keyspace, edgeShardSerialization );
+//
+//
+// DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId("source"), "test" );
+//
+//
+// /**
+// * Mock up returning edges from the source
+// */
+//
+// int count = 100;
+//
+// for(int i = 0; i < count; i ++){
+//
+//
+//
+// when(shardedEdgeSerialization.getEdgesFromSource( same(edgeColumnFamilies), same(scope), any(
+// SearchByEdgeType.class), Matchers.argThat(new ShardSetMatcher( Collections.singleton( sourceShard ) ))/*any(Set.class)*/ ));
+// edgeMeta.loadEdges( shardedEdgeSerialization, edgeColumnFamilies, scope,
+//
+// Collections.singleton( sourceShard ), SearchByEdgeType.Order.DESCENDING, Long.MAX_VALUE );
+// }
+//
+// try {
+// compaction.compact( this.scope, directedEdgeMeta, group );
+// fail( "I should not reach this point" );
+// }catch(Throwable t){
+// assertEquals("Correct error message returned", "Compaction cannot be run yet. Ignoring compaction.", t.getMessage());
+// }
+//
+// }
+
+
+ private final class ShardSetMatcher extends BaseMatcher<Collection<Shard>>{
+
+ private final Collection<Shard> expected;
+
+
+ private ShardSetMatcher( final Collection<Shard> expected ) {this.expected = expected;}
+
+
+ @Override
+ public boolean matches( final Object o ) {
+ if(! (o instanceof Collection)){
+ return false;
+ }
+
+
+ Collection<Shard> passedShards = ( Collection<Shard> ) o;
+
+ return passedShards.containsAll( expected );
+ }
+
+
+ @Override
+ public void describeTo( final Description description ) {
+
+ StringBuilder builder = new StringBuilder( );
+
+ builder.append("Collection of shards with shards {");
+
+ for(Shard shard: expected){
+ builder.append(shard).append( "," );
+ }
+
+ builder.setLength( builder.length()-1 );
+
+ description.appendText( builder.toString() );
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
index 8e9ed5c..da96e33 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardApproximationTest.java
@@ -489,6 +489,18 @@ public class NodeShardApproximationTest {
@Override
+ public int getShardAuditWorkerCount() {
+ return 0;
+ }
+
+
+ @Override
+ public int getShardAuditWorkerQueueSize() {
+ return 0;
+ }
+
+
+ @Override
public long getCounterFlushCount() {
return 100000l;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
index 5b20647..34bc079 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/ShardEntryGroupIteratorTest.java
@@ -29,62 +29,86 @@ import java.util.Iterator;
import org.junit.Test;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.Shard;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardEntryGroup;
+import org.apache.usergrid.persistence.graph.serialization.impl.shard.ShardGroupCompaction;
import static junit.framework.TestCase.assertTrue;
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Matchers.same;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
public class ShardEntryGroupIteratorTest {
+
@Test(expected = IllegalArgumentException.class)
- public void noShards(){
+ public void noShards() {
+
+ final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
+ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId( "source" ), "test" );
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
final long delta = 10000;
final Iterator<Shard> noShards = Collections.<Shard>emptyList().iterator();
//should blow up, our iterator is empty
- new ShardEntryGroupIterator(noShards, delta);
-
+ new ShardEntryGroupIterator( noShards, delta, shardGroupCompaction, scope, directedEdgeMeta );
}
+
@Test
- public void existingSingleShard(){
+ public void existingSingleShard() {
- final Shard minShard = new Shard(0, 0, true);
+ final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
+ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId( "source" ), "test" );
+
+
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
+ final Shard minShard = new Shard( 0, 0, true );
final long delta = 10000;
final Iterator<Shard> noShards = Collections.singleton( minShard ).iterator();
- ShardEntryGroupIterator entryGroupIterator = new ShardEntryGroupIterator(noShards, delta);
+ ShardEntryGroupIterator entryGroupIterator =
+ new ShardEntryGroupIterator( noShards, delta, shardGroupCompaction, scope, directedEdgeMeta );
+
- assertTrue("Root shard always present", entryGroupIterator.hasNext());
+ assertTrue( "Root shard always present", entryGroupIterator.hasNext() );
ShardEntryGroup group = entryGroupIterator.next();
- assertNotNull("Group returned", group);
+ assertNotNull( "Group returned", group );
- Collection<Shard> readShards = group.getReadShards();
+ //verify we ran our compaction check
+ verify( shardGroupCompaction ).evaluateShardGroup( same( scope ), same( directedEdgeMeta ), eq( group ) );
- assertEquals("Min shard present", 1, readShards.size());
- assertTrue("Min shard present", readShards.contains( minShard ));
+ Collection<Shard> readShards = group.getReadShards( );
+ assertEquals( "Min shard present", 1, readShards.size() );
- Collection<Shard> writeShards = group.getWriteShards( 0 );
+ assertTrue( "Min shard present", readShards.contains( minShard ) );
- assertEquals("Min shard present", 1, writeShards.size());
- assertTrue("Min shard present", writeShards.contains( minShard ));
+ Collection<Shard> writeShards = group.getWriteShards( 0 );
+ assertEquals( "Min shard present", 1, writeShards.size() );
- writeShards = group.getWriteShards( Long.MAX_VALUE );
+ assertTrue( "Min shard present", writeShards.contains( minShard ) );
- assertEquals("Min shard present", 1, writeShards.size());
- assertTrue("Min shard present", writeShards.contains( minShard ));
+ writeShards = group.getWriteShards( Long.MAX_VALUE );
+ assertEquals( "Min shard present", 1, writeShards.size() );
+ assertTrue( "Min shard present", writeShards.contains( minShard ) );
}
@@ -93,140 +117,160 @@ public class ShardEntryGroupIteratorTest {
* that only the last 1 or 2 groups will actually have more than 1 entry.
*/
@Test
- public void boundedShardSets(){
+ public void boundedShardSets() {
+
+ final ApplicationScope scope = new ApplicationScopeImpl( createId( "application" ) );
+ final DirectedEdgeMeta directedEdgeMeta = DirectedEdgeMeta.fromSourceNode( createId( "source" ), "test" );
+
+ final ShardGroupCompaction shardGroupCompaction = mock( ShardGroupCompaction.class );
+
/**
* Next shard group
*/
- final Shard shardGroup1Shard1 = new Shard(0, 0, true);
+ final Shard shardGroup1Shard1 = new Shard( 0, 0, true );
- final Shard shardGroup1Shard2 = new Shard(10000, 100, false);
+ final Shard shardGroup1Shard2 = new Shard( 10000, 100, false );
- final Shard shardGroup1Shard3 = new Shard(20000, 200, false);
+ final Shard shardGroup1Shard3 = new Shard( 20000, 200, false );
/**
* Middle shard group
*/
- final Shard shardGroup2Shard1 = new Shard(30000, 300, true);
+ final Shard shardGroup2Shard1 = new Shard( 30000, 300, true );
- final Shard shardGroup2Shard2 = new Shard(40000, 400, false);
+ final Shard shardGroup2Shard2 = new Shard( 40000, 400, false );
/**
* Highest shard group
*/
- final Shard shardGroup3Shard1 = new Shard(50000, 500, true);
-
- final Shard shardGroup3Shard2 = new Shard(60000, 600, false);
-
- final Shard shardGroup3Shard3 = new Shard(70000, 700, false);
+ final Shard shardGroup3Shard1 = new Shard( 50000, 500, true );
+ final Shard shardGroup3Shard2 = new Shard( 60000, 600, false );
+ final Shard shardGroup3Shard3 = new Shard( 70000, 700, false );
final long delta = 10000;
- final Iterator<Shard> noShards = Arrays.asList(shardGroup3Shard3, shardGroup3Shard2, shardGroup3Shard1, shardGroup2Shard2, shardGroup2Shard1, shardGroup1Shard3, shardGroup1Shard2, shardGroup1Shard1 ).iterator();
-
+ final Iterator<Shard> noShards =
+ Arrays.asList( shardGroup3Shard3, shardGroup3Shard2, shardGroup3Shard1, shardGroup2Shard2,
+ shardGroup2Shard1, shardGroup1Shard3, shardGroup1Shard2, shardGroup1Shard1 ).iterator();
- ShardEntryGroupIterator entryGroupIterator = new ShardEntryGroupIterator(noShards, delta);
+ ShardEntryGroupIterator entryGroupIterator =
+ new ShardEntryGroupIterator( noShards, delta, shardGroupCompaction, scope, directedEdgeMeta );
- assertTrue("max group present", entryGroupIterator.hasNext());
+ assertTrue( "max group present", entryGroupIterator.hasNext() );
ShardEntryGroup group = entryGroupIterator.next();
- assertNotNull("Group returned", group);
+ assertNotNull( "Group returned", group );
- Collection<Shard> readShards = group.getReadShards();
+ //verify we ran our compaction check
+ verify( shardGroupCompaction ).evaluateShardGroup( same( scope ), same( directedEdgeMeta ), eq( group ) );
- assertEquals("Min shard present", 3, readShards.size());
+ Collection<Shard> readShards = group.getReadShards( );
- assertTrue("shardGroup3Shard3 shard present", readShards.contains( shardGroup3Shard3 ));
+ assertEquals( "Both shards present", 2, readShards.size() );
- assertTrue("shardGroup3Shard2 shard present", readShards.contains( shardGroup3Shard2 ));
+ assertTrue( "shardGroup3Shard2 shard present", readShards.contains( shardGroup3Shard2 ) );
- assertTrue("shardGroup3Shard1 shard present", readShards.contains( shardGroup3Shard1 ));
+ assertTrue( "shardGroup3Shard1 shard present", readShards.contains( shardGroup3Shard1 ) );
Collection<Shard> writeShards = group.getWriteShards( 0 );
- assertEquals("Min shard present", 3, writeShards.size());
+ assertEquals( "Min shard present", 1, writeShards.size() );
- assertTrue("shardGroup3Shard3 shard present", writeShards.contains( shardGroup3Shard3 ));
- assertTrue("shardGroup3Shard2 shard present", writeShards.contains( shardGroup3Shard2 ));
+ assertTrue( "shardGroup3Shard1 shard present", writeShards.contains( shardGroup3Shard1 ) );
- assertTrue("shardGroup3Shard1 shard present", writeShards.contains( shardGroup3Shard1 ));
+ writeShards = group.getWriteShards( shardGroup3Shard3.getCreatedTime() + delta );
+ assertEquals( "Min shard present", 1, writeShards.size() );
+ assertTrue( "shardGroup3Shard2 shard present", readShards.contains( shardGroup3Shard2 ) );
+ assertTrue( "shardGroup3Shard1 shard present", writeShards.contains( shardGroup3Shard1 ) );
- assertTrue("middle group present", entryGroupIterator.hasNext());
- group = entryGroupIterator.next();
+ /****
+ * Middle group
+ */
- assertNotNull("Group returned", group);
+ assertTrue( "middle group present", entryGroupIterator.hasNext() );
- readShards = group.getReadShards();
+ group = entryGroupIterator.next();
- assertEquals("Min shard present", 2, readShards.size());
+ assertNotNull( "Group returned", group );
- assertTrue("shardGroup2Shard1 shard present", readShards.contains( shardGroup2Shard1 ));
+ //verify we ran our compaction check
+ verify( shardGroupCompaction ).evaluateShardGroup( same( scope ), same( directedEdgeMeta ), eq( group ) );
- assertTrue("shardGroup2Shard2 shard present", readShards.contains( shardGroup2Shard2 ));
+ readShards = group.getReadShards( );
- writeShards = group.getWriteShards( 0 );
+ assertEquals( "Both shards present", 2, readShards.size() );
- assertEquals("Min shard present", 2, writeShards.size());
+ assertTrue( "shardGroup2Shard1 shard present", readShards.contains( shardGroup2Shard1 ) );
- assertTrue("shardGroup2Shard1 shard present", writeShards.contains( shardGroup2Shard1 ));
+ assertTrue( "shardGroup2Shard2 shard present", readShards.contains( shardGroup2Shard2 ) );
- assertTrue("shardGroup2Shard2 shard present", writeShards.contains( shardGroup2Shard2 ));
+ writeShards = group.getWriteShards( 0 );
+ assertEquals( "Min shard present", 1, writeShards.size() );
+ assertTrue( "shardGroup2Shard1 shard present", writeShards.contains( shardGroup2Shard1 ) );
+ writeShards = group.getWriteShards( shardGroup2Shard2.getCreatedTime() + delta +1 );
- assertTrue("min group present", entryGroupIterator.hasNext());
+ assertEquals( "Both shards present", 1, writeShards.size() );
- group = entryGroupIterator.next();
+ assertTrue( "shardGroup2Shard2 shard present", writeShards.contains( shardGroup2Shard2 ) );
- assertNotNull("Group returned", group);
- readShards = group.getReadShards();
+ /*****
+ * Minimum group
+ */
- assertEquals("Min shard present", 3, readShards.size());
+ assertTrue( "min group present", entryGroupIterator.hasNext() );
- assertTrue("shardGroup1Shard3 shard present", readShards.contains( shardGroup1Shard3 ));
+ group = entryGroupIterator.next();
- assertTrue("shardGroup1Shard2 shard present", readShards.contains( shardGroup1Shard2 ));
+ assertNotNull( "Group returned", group );
- assertTrue("shardGroup1Shard1 shard present", readShards.contains( shardGroup1Shard1 ));
+ //verify we ran our compaction check
+ verify( shardGroupCompaction ).evaluateShardGroup( same( scope ), same( directedEdgeMeta ), eq( group ) );
- writeShards = group.getWriteShards( 0 );
+ readShards = group.getReadShards();
- assertEquals("Min shard present", 3, writeShards.size());
+ assertEquals( "Both shards present", 2, readShards.size() );
- assertTrue("shardGroup1Shard3 shard present", writeShards.contains( shardGroup1Shard3 ));
+ assertTrue( "shardGroup1Shard1 shard present", readShards.contains( shardGroup1Shard1 ) );
+ assertTrue( "shardGroup1Shard2 shard present", readShards.contains( shardGroup1Shard2 ) );
- assertTrue("shardGroup1Shard2 shard present", writeShards.contains( shardGroup1Shard2 ));
- assertTrue("shardGroup1Shard1 shard present", writeShards.contains( shardGroup1Shard1 ));
+ writeShards = group.getWriteShards( 0 );
+ assertEquals( "Min shard present", 1, writeShards.size() );
+ assertTrue( "shardGroup1Shard1 shard present", writeShards.contains( shardGroup1Shard1 ) );
+ writeShards = group.getWriteShards( shardGroup1Shard3.getCreatedTime() + delta + 1 );
+ assertEquals( "Both shards present", 1, writeShards.size() );
+ assertTrue( "shardGroup1Shard2 shard present", writeShards.contains( shardGroup1Shard2 ) );
}
-
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java
new file mode 100644
index 0000000..d0adc1e
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/SourceDirectedEdgeDescendingComparatorTest.java
@@ -0,0 +1,136 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators;
+
+
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class SourceDirectedEdgeDescendingComparatorTest {
+
+ final SourceDirectedEdgeDescendingComparator comp = SourceDirectedEdgeDescendingComparator.INSTANCE;
+
+
+ @Test
+ public void sameEdges() {
+
+ final Id sourceId = createId( "source" );
+ final Id targetId = createId( "target" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp, true );
+
+
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertEquals( 0, compare );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertEquals( 0, compare );
+ }
+
+
+ @Test
+ public void timestampDifferent() {
+
+ final Id sourceId = createId( "source" );
+ final Id targetId = createId( "target" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp + 1, true );
+
+
+ //marked edge 1 is less than timestamp, it should be considered "greater"
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertEquals( 1, compare );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertEquals( -1, compare );
+ }
+
+
+ @Test
+ public void uuidDifferent() {
+
+ final Id sourceId1 = createId( "source" );
+ final Id sourceId2 = createId( "source" );
+ final Id targetId = createId( "target" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId1, type, targetId, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId2, type, targetId, timestamp, true );
+
+
+ //marked edge 1 uuid is a is less than target uuid timestamp, it should be considered "greater"
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertTrue( compare > 0 );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertTrue( compare < 0 );
+ }
+
+
+ @Test
+ public void idTypeDifferent() {
+
+ final UUID sourceId = UUIDGenerator.newTimeUUID();
+
+ final Id sourceId1 = createId( sourceId, "source1" );
+ final Id sourceId2 = createId( sourceId, "source2" );
+ final Id targetId = createId( "target" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId2, type, targetId, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId1, type, targetId, timestamp, true );
+
+
+ //marked edge 1 is less than timestamp, it should be considered "greater"
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertEquals( 1, compare );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertEquals( -1, compare );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java
new file mode 100644
index 0000000..15df661
--- /dev/null
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/comparators/TargetDirectedEdgeDescendingComparatorTest.java
@@ -0,0 +1,136 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.shard.impl.comparators;
+
+
+import java.util.UUID;
+
+import org.junit.Test;
+
+import org.apache.usergrid.persistence.graph.impl.SimpleMarkedEdge;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
+
+import static org.apache.usergrid.persistence.graph.test.util.EdgeTestUtils.createId;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+
+public class TargetDirectedEdgeDescendingComparatorTest {
+
+ final TargetDirectedEdgeDescendingComparator comp = TargetDirectedEdgeDescendingComparator.INSTANCE;
+
+
+ @Test
+ public void sameEdges() {
+
+ final Id sourceId = createId( "source" );
+ final Id targetId = createId( "target" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp, true );
+
+
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertEquals( 0, compare );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertEquals( 0, compare );
+ }
+
+
+ @Test
+ public void timestampDifferent() {
+
+ final Id sourceId = createId( "source" );
+ final Id targetId = createId( "target" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId, type, targetId, timestamp + 1, true );
+
+
+ //marked edge 1 is less than timestamp, it should be considered "greater"
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertEquals( 1, compare );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertEquals( -1, compare );
+ }
+
+
+ @Test
+ public void uuidDifferent() {
+
+ final Id sourceId = createId( "source" );
+ final Id targetId1 = createId( "target" );
+ final Id targetId2 = createId( "target" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId, type, targetId1, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId, type, targetId2, timestamp, true );
+
+
+ //marked edge 1 uuid is a is less than target uuid timestamp, it should be considered "greater"
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertTrue( compare > 0 );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertTrue( compare < 0 );
+ }
+
+
+ @Test
+ public void idTypeDifferent() {
+
+ final UUID targetId = UUIDGenerator.newTimeUUID();
+
+ final Id sourceId = createId( "source" );
+ final Id targetId1 = createId( targetId, "target1" );
+ final Id targetId2 = createId( targetId, "target2" );
+ final String type = "type";
+ final long timestamp = 10000;
+
+ final SimpleMarkedEdge markedEdge1 = new SimpleMarkedEdge( sourceId, type, targetId2, timestamp, true );
+ final SimpleMarkedEdge markedEdge2 = new SimpleMarkedEdge( sourceId, type, targetId1, timestamp, true );
+
+
+ //marked edge 1 is less than timestamp, it should be considered "greater"
+ int compare = comp.compare( markedEdge1, markedEdge2 );
+
+ assertEquals( 1, compare );
+
+ compare = comp.compare( markedEdge2, markedEdge1 );
+
+ assertEquals( -1, compare );
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
index c9507d4..ca82b8d 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/test/util/EdgeTestUtils.java
@@ -168,7 +168,7 @@ public class EdgeTestUtils {
*/
public static SearchByEdgeType createSearchByEdge( final Id sourceId, final String type, final long maxVersion,
final Edge last ) {
- return new SimpleSearchByEdgeType( sourceId, type, maxVersion, last );
+ return new SimpleSearchByEdgeType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, last );
}
@@ -183,7 +183,7 @@ public class EdgeTestUtils {
*/
public static SearchByIdType createSearchByEdgeAndId( final Id sourceId, final String type, final long maxVersion,
final String idType, final Edge last ) {
- return new SimpleSearchByIdType( sourceId, type, maxVersion, idType, last );
+ return new SimpleSearchByIdType( sourceId, type, maxVersion, SearchByEdgeType.Order.DESCENDING, idType, last );
}
@@ -211,7 +211,7 @@ public class EdgeTestUtils {
*/
public static SearchByEdge createGetByEdge( final Id sourceId, final String type, final Id targetId,
final long maxVersion, final Edge last ) {
- return new SimpleSearchByEdge( sourceId, type, targetId, maxVersion, last );
+ return new SimpleSearchByEdge( sourceId, type, targetId, maxVersion, SearchByEdgeType.Order.DESCENDING, last );
}
//
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/log4j.properties b/stack/corepersistence/graph/src/test/resources/log4j.properties
index 08d897c..f9ea207 100644
--- a/stack/corepersistence/graph/src/test/resources/log4j.properties
+++ b/stack/corepersistence/graph/src/test/resources/log4j.properties
@@ -35,5 +35,6 @@ log4j.logger.cassandra.db=ERROR
log4j.logger.org.apache.usergrid.persistence.graph=TRACE
log4j.logger.org.apache.usergrid.persistence.core.rx=TRACE
+log4j.logger.org.apache.usergrid.persistence.core.astyanax=TRACE
#log4j.logger.org.apache.usergrid.persistence.graph.serialization.impl.parse=TRACE
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e040fdf4/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
index 61612f6..1cb970f 100644
--- a/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/graph/src/test/resources/usergrid-UNIT.properties
@@ -13,5 +13,7 @@ collections.keyspace.strategy.options=replication_factor:1
collections.keyspace.strategy.class=SimpleStrategy
collection.stage.transient.timeout=60
+usergrid.graph.shard.repair.chance=.20
+
hystrix.threadpool.graph_user.coreSize=8
hystrix.threadpool.graph_async.coreSize=8