You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/28 00:04:44 UTC
[5/7] git commit: Finished the migration of map keys to buckets
Finished the migration of map keys to buckets
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6b203de0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6b203de0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6b203de0
Branch: refs/heads/key-row-sharding
Commit: 6b203de04bf8bbe4279fd619e096d8d260e9851c
Parents: dd84d6b
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Oct 24 12:25:55 2014 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Oct 24 12:25:55 2014 -0600
----------------------------------------------------------------------
.../core/astyanax/BucketScopedRowKey.java | 13 ++-
.../astyanax/BucketScopedRowKeySerializer.java | 94 ++++++++++++++++++++
...rganizationScopedBucketRowKeySerializer.java | 93 -------------------
.../OrganizationScopedRowKeySerializer.java | 87 ------------------
.../core/astyanax/ScopedRowKeySerializer.java | 87 ++++++++++++++++++
.../core/hash/ExpandingBucketLocator.java | 42 ++++-----
.../core/hash/ExpandingBucketLocatorTest.java | 9 +-
.../impl/EdgeMetadataSerializationImpl.java | 10 +--
.../impl/NodeSerializationImpl.java | 6 +-
.../NodeShardCounterSerializationImpl.java | 4 +-
.../shard/impl/EdgeShardSerializationImpl.java | 4 +-
.../shard/impl/SizebasedEdgeColumnFamilies.java | 13 ++-
.../map/impl/MapSerializationImpl.java | 41 +++++----
13 files changed, 256 insertions(+), 247 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKey.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKey.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKey.java
index 0687a7a..fe04875 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKey.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKey.java
@@ -25,6 +25,11 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.model.entity.Id;
+/**
+ * A scoped row key that also includes an index, which can be used for consistent hashing.
+ *
+ * @param <K>
+ */
public class BucketScopedRowKey< K> extends ScopedRowKey< K> {
private final int bucketNumber;
@@ -100,13 +105,13 @@ public class BucketScopedRowKey< K> extends ScopedRowKey< K> {
/**
* Create a list of all buckets from [0, totalBuckets}. Note that this is an n-1 0 based system
*/
- public static < K> List<BucketScopedRowKey< K>> fromRange( final Id scope, final K key, final int totalBuckets ) {
+ public static < K> List<BucketScopedRowKey< K>> fromRange( final Id scope, final K key, final int... buckets ) {
- final List<BucketScopedRowKey< K>> results = new ArrayList<>( totalBuckets );
+ final List<BucketScopedRowKey< K>> results = new ArrayList<>( buckets.length );
- for ( int i = 0; i < totalBuckets; i++ ) {
- results.add( new BucketScopedRowKey<>( scope, key, i ) );
+ for ( int i = 0; i < buckets.length; i++ ) {
+ results.add( new BucketScopedRowKey<>( scope, key, buckets[i] ) );
}
return results;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKeySerializer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKeySerializer.java
new file mode 100644
index 0000000..401cc83
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/BucketScopedRowKeySerializer.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+import com.netflix.astyanax.model.Composites;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+
+
+/**
+ * Serializer for serializing CollectionScope + any type into row keys.
+ *
+ *
+ * @author tnine
+ */
+public class BucketScopedRowKeySerializer<K> extends AbstractSerializer<BucketScopedRowKey<K>> {
+
+
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ /**
+ * The delegate serializer for the key
+ */
+ private final CompositeFieldSerializer<K> keySerializer;
+
+
+
+
+ public BucketScopedRowKeySerializer( final CompositeFieldSerializer<K> keySerializer ) {
+ this.keySerializer = keySerializer;
+ }
+
+
+ @Override
+ public ByteBuffer toByteBuffer( final BucketScopedRowKey<K> scopedRowKey ) {
+
+ final CompositeBuilder builder = Composites.newCompositeBuilder();
+
+ //add the organization's id
+ ID_SER.toComposite( builder, scopedRowKey.getScope() );
+
+ //add the bucket
+ builder.addInteger( scopedRowKey.getBucketNumber() );
+
+ //add the key type
+ keySerializer.toComposite( builder, scopedRowKey.getKey() );
+
+ return builder.build();
+ }
+
+
+ @Override
+ public BucketScopedRowKey<K> fromByteBuffer( final ByteBuffer byteBuffer ) {
+ final CompositeParser parser = Composites.newCompositeParser( byteBuffer );
+
+ //read back the id
+ final Id orgId = ID_SER.fromComposite( parser );
+
+ final int bucket = parser.readInteger();
+
+ final K value = keySerializer.fromComposite( parser );
+
+ return new BucketScopedRowKey<>( orgId, value, bucket );
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedBucketRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedBucketRowKeySerializer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedBucketRowKeySerializer.java
deleted file mode 100644
index 8fddedd..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedBucketRowKeySerializer.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.core.astyanax;
-
-
-import java.nio.ByteBuffer;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-import com.netflix.astyanax.model.Composites;
-import com.netflix.astyanax.serializers.AbstractSerializer;
-import com.netflix.astyanax.serializers.IntegerSerializer;
-
-
-/**
- * Serializer for serializing CollectionScope + any type into row keys
- *
- * @author tnine
- */
-public class OrganizationScopedBucketRowKeySerializer<K> extends AbstractSerializer<BucketScopedRowKey<K>> {
-
-
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
-
- /**
- * The delegate serializer for the key
- */
- private final CompositeFieldSerializer<K> keySerializer;
-
-
-
-
- public OrganizationScopedBucketRowKeySerializer( final CompositeFieldSerializer<K> keySerializer ) {
- this.keySerializer = keySerializer;
- }
-
-
- @Override
- public ByteBuffer toByteBuffer( final BucketScopedRowKey<K> scopedRowKey ) {
-
- final CompositeBuilder builder = Composites.newCompositeBuilder();
-
- //add the organization's id
- ID_SER.toComposite( builder, scopedRowKey.getScope() );
-
- //add the bucket
- builder.addInteger( scopedRowKey.getBucketNumber() );
-
- //add the key type
- keySerializer.toComposite( builder, scopedRowKey.getKey() );
-
- return builder.build();
- }
-
-
- @Override
- public BucketScopedRowKey<K> fromByteBuffer( final ByteBuffer byteBuffer ) {
- final CompositeParser parser = Composites.newCompositeParser( byteBuffer );
-
- //read back the id
- final Id orgId = ID_SER.fromComposite( parser );
-
- final int bucket = parser.readInteger();
-
- final K value = keySerializer.fromComposite( parser );
-
- return new BucketScopedRowKey<>( orgId, value, bucket );
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedRowKeySerializer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedRowKeySerializer.java
deleted file mode 100644
index 4289628..0000000
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/OrganizationScopedRowKeySerializer.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.usergrid.persistence.core.astyanax;
-
-
-import java.nio.ByteBuffer;
-
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
-import org.apache.usergrid.persistence.model.entity.Id;
-
-import com.netflix.astyanax.model.CompositeBuilder;
-import com.netflix.astyanax.model.CompositeParser;
-import com.netflix.astyanax.model.Composites;
-import com.netflix.astyanax.serializers.AbstractSerializer;
-
-
-/**
- * Serializer for serializing CollectionScope + any type into row keys
- *
- * @author tnine
- */
-public class OrganizationScopedRowKeySerializer<K> extends AbstractSerializer<ScopedRowKey<K>> {
-
-
- private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
-
-
- /**
- * The delegate serializer for the key
- */
- private final CompositeFieldSerializer<K> keySerializer;
-
-
-
-
- public OrganizationScopedRowKeySerializer( final CompositeFieldSerializer<K> keySerializer ) {
- this.keySerializer = keySerializer;
- }
-
-
- @Override
- public ByteBuffer toByteBuffer( final ScopedRowKey< K> scopedRowKey ) {
-
- final CompositeBuilder builder = Composites.newCompositeBuilder();
-
- //add the organization's id
- ID_SER.toComposite( builder, scopedRowKey.getScope() );
-
- //add the key type
- keySerializer.toComposite( builder, scopedRowKey.getKey() );
-
- return builder.build();
- }
-
-
- @Override
- public ScopedRowKey<K> fromByteBuffer( final ByteBuffer byteBuffer ) {
- final CompositeParser parser = Composites.newCompositeParser( byteBuffer );
-
- //read back the id
- final Id orgId = ID_SER.fromComposite( parser );
-
- final K value = keySerializer.fromComposite( parser );
-
- return new ScopedRowKey<K>( orgId, value );
- }
-}
-
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ScopedRowKeySerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ScopedRowKeySerializer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ScopedRowKeySerializer.java
new file mode 100644
index 0000000..30e1792
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/ScopedRowKeySerializer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.model.entity.Id;
+
+import com.netflix.astyanax.model.CompositeBuilder;
+import com.netflix.astyanax.model.CompositeParser;
+import com.netflix.astyanax.model.Composites;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+
+
+/**
+ * Serializer for serializing CollectionScope + any type into row keys
+ *
+ * @author tnine
+ */
+public class ScopedRowKeySerializer<K> extends AbstractSerializer<ScopedRowKey<K>> {
+
+
+ private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
+
+
+ /**
+ * The delegate serializer for the key
+ */
+ private final CompositeFieldSerializer<K> keySerializer;
+
+
+
+
+ public ScopedRowKeySerializer( final CompositeFieldSerializer<K> keySerializer ) {
+ this.keySerializer = keySerializer;
+ }
+
+
+ @Override
+ public ByteBuffer toByteBuffer( final ScopedRowKey< K> scopedRowKey ) {
+
+ final CompositeBuilder builder = Composites.newCompositeBuilder();
+
+ //add the organization's id
+ ID_SER.toComposite( builder, scopedRowKey.getScope() );
+
+ //add the key type
+ keySerializer.toComposite( builder, scopedRowKey.getKey() );
+
+ return builder.build();
+ }
+
+
+ @Override
+ public ScopedRowKey<K> fromByteBuffer( final ByteBuffer byteBuffer ) {
+ final CompositeParser parser = Composites.newCompositeParser( byteBuffer );
+
+ //read back the id
+ final Id orgId = ID_SER.fromComposite( parser );
+
+ final K value = keySerializer.fromComposite( parser );
+
+ return new ScopedRowKey<K>( orgId, value );
+ }
+}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocator.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocator.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocator.java
index c9bba2f..3d81389 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocator.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocator.java
@@ -39,26 +39,19 @@ package org.apache.usergrid.persistence.core.hash;/*
*/
-import java.util.ArrayList;
-import java.util.List;
-
import com.google.common.hash.Funnel;
/**
- * An algorithm that will generate all possible keys for different "Levels" of sharding. For instance, imagine this scheme.
+ * An algorithm that will generate all possible keys for different "Levels" of sharding. For instance, imagine this
+ * scheme.
*
- * 1 Shard
- * 2 Shards
- * 4 Shards
- * 8 Shards
+ * 1 Shard 2 Shards 4 Shards 8 Shards
*
* (note that we do not need to expand by 2x each time, this is merely an example).
*
- * When seeking on a string key, for 4 levels of the key, we get 4 different keys due to different shard sizes.
- * This is faster than seeking ALL shards, since this would result in 15 shards, vs 4 in the example.
- *
- * @param <T>
+ * When seeking on a string key, for 4 levels of the key, we get 4 different keys due to different shard sizes. This is
+ * faster than seeking ALL shards, since this would result in 15 shards, vs 4 in the example.
*/
public class ExpandingBucketLocator<T> {
@@ -66,34 +59,37 @@ public class ExpandingBucketLocator<T> {
/**
- * Create a new instance with the specified history. For instance, from the javadoc above, the constructor
- * would contains {8, 4, 3, 2, 1}. Shards are returned in the size order they are given in the constructor
- * @param funnel
- * @param bucketSizes
+ * Create a new instance with the specified history. For instance, from the javadoc above, the constructor would
+ * contains {8, 4, 3, 2, 1}. Shards are returned in the size order they are given in the constructor
*/
public ExpandingBucketLocator( final Funnel<T> funnel, final int... bucketSizes ) {
bucketLocatorList = new BucketLocator[bucketSizes.length];
- for(int i = 0; i < bucketSizes.length; i ++){
+ for ( int i = 0; i < bucketSizes.length; i++ ) {
bucketLocatorList[i] = new BucketLocator<>( funnel, bucketSizes[i] );
}
-
}
/**
* Hash the results, and return them in the same order as specified in the constructor
- * @param hash
- * @return
*/
- public int[] getBuckets(T hash){
+ public int[] getAllBuckets( T hash ) {
int[] results = new int[bucketLocatorList.length];
- for(int i = 0; i < bucketLocatorList.length; i ++){
- results[i] = bucketLocatorList[i].getBucket( hash );
+ for ( int i = 0; i < bucketLocatorList.length; i++ ) {
+ results[i] = bucketLocatorList[i].getBucket( hash );
}
return results;
}
+
+
+ /**
+ * Get the current bucket for the hash value. Hashes from the first element in the list
+ */
+ public int getCurrentBucket( T hash ) {
+ return bucketLocatorList[0].getBucket( hash );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocatorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocatorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocatorTest.java
index 32deb07..ce691fc 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocatorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/hash/ExpandingBucketLocatorTest.java
@@ -44,8 +44,8 @@ public class ExpandingBucketLocatorTest extends TestCase {
final String key = "mytestkey";
- int[] results1 = expandingBucketLocator1.getBuckets(key );
- int[] results2 = expandingBucketLocator2.getBuckets(key );
+ int[] results1 = expandingBucketLocator1.getAllBuckets(key );
+ int[] results2 = expandingBucketLocator2.getAllBuckets(key );
assertTrue( "Same results returned", Arrays.equals(results1, results2));
@@ -53,6 +53,11 @@ public class ExpandingBucketLocatorTest extends TestCase {
assertTrue("Within bounds", results1[1] <= 9);
assertTrue("Within bounds", results1[2] <= 0);
+ //test the first hash
+ int newestBucket = expandingBucketLocator1.getCurrentBucket( key );
+
+ assertEquals("Same bucket returned", results1[0], newestBucket);
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationImpl.java
index 131e426..6c20cfe 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/EdgeMetadataSerializationImpl.java
@@ -33,7 +33,7 @@ import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.astyanax.StringColumnParser;
import org.apache.usergrid.persistence.core.migration.Migration;
@@ -70,14 +70,14 @@ public class EdgeMetadataSerializationImpl implements EdgeMetadataSerialization,
//row key serializers
private static final IdRowCompositeSerializer ID_SER = IdRowCompositeSerializer.get();
- private static final OrganizationScopedRowKeySerializer<Id> ROW_KEY_SER =
- new OrganizationScopedRowKeySerializer<Id>( ID_SER );
+ private static final ScopedRowKeySerializer<Id> ROW_KEY_SER =
+ new ScopedRowKeySerializer<Id>( ID_SER );
private static final StringSerializer STRING_SERIALIZER = StringSerializer.get();
private static final EdgeTypeRowCompositeSerializer EDGE_SER = new EdgeTypeRowCompositeSerializer();
- private static final OrganizationScopedRowKeySerializer<EdgeIdTypeKey> EDGE_TYPE_ROW_KEY =
- new OrganizationScopedRowKeySerializer<EdgeIdTypeKey>( EDGE_SER );
+ private static final ScopedRowKeySerializer<EdgeIdTypeKey> EDGE_TYPE_ROW_KEY =
+ new ScopedRowKeySerializer<EdgeIdTypeKey>( EDGE_SER );
private static final StringColumnParser PARSER = StringColumnParser.get();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
index 0febf96..eae7fa0 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/NodeSerializationImpl.java
@@ -36,14 +36,13 @@ import org.apache.usergrid.persistence.core.astyanax.CassandraConfig;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.migration.Migration;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.exception.GraphRuntimeException;
import org.apache.usergrid.persistence.graph.serialization.NodeSerialization;
import org.apache.usergrid.persistence.graph.serialization.util.GraphValidation;
import org.apache.usergrid.persistence.model.entity.Id;
@@ -53,7 +52,6 @@ import com.google.common.base.Preconditions;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.MutationBatch;
-import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
import com.netflix.astyanax.connectionpool.exceptions.NotFoundException;
import com.netflix.astyanax.model.Column;
import com.netflix.astyanax.model.Row;
@@ -90,7 +88,7 @@ public class NodeSerializationImpl implements NodeSerialization, Migration {
*/
private static final MultiTennantColumnFamily<ScopedRowKey<Id>, Boolean> GRAPH_DELETE =
new MultiTennantColumnFamily<>( "Graph_Marked_Nodes",
- new OrganizationScopedRowKeySerializer<Id>( ROW_SERIALIZER ), BOOLEAN_SERIALIZER );
+ new ScopedRowKeySerializer<Id>( ROW_SERIALIZER ), BOOLEAN_SERIALIZER );
protected final Keyspace keyspace;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
index 7c267a0..524a0cf 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationImpl.java
@@ -33,7 +33,7 @@ import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
import org.apache.usergrid.persistence.core.astyanax.IdRowCompositeSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.hystrix.HystrixCassandra;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -68,7 +68,7 @@ public class NodeShardCounterSerializationImpl implements NodeShardCounterSerial
*/
private static final MultiTennantColumnFamily<ScopedRowKey<ShardKey>, Boolean> EDGE_SHARD_COUNTS =
new MultiTennantColumnFamily<>( "Edge_Shard_Counts",
- new OrganizationScopedRowKeySerializer<>( SHARD_KEY_SERIALIZER ), BooleanSerializer.get() );
+ new ScopedRowKeySerializer<>( SHARD_KEY_SERIALIZER ), BooleanSerializer.get() );
protected final Keyspace keyspace;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
index 4fd3155..f107307 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/EdgeShardSerializationImpl.java
@@ -32,7 +32,7 @@ import org.apache.usergrid.persistence.core.astyanax.ColumnParser;
import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -63,7 +63,7 @@ public class EdgeShardSerializationImpl implements EdgeShardSerialization {
*/
private static final MultiTennantColumnFamily<ScopedRowKey<DirectedEdgeMeta>, Long> EDGE_SHARDS =
new MultiTennantColumnFamily<>( "Edge_Shards",
- new OrganizationScopedRowKeySerializer<>( EdgeShardRowKeySerializer.INSTANCE ), LongSerializer.get() );
+ new ScopedRowKeySerializer<>( EdgeShardRowKeySerializer.INSTANCE ), LongSerializer.get() );
private static final ShardColumnParser COLUMN_PARSER = new ShardColumnParser();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
index b7d5636..8f16448 100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/impl/SizebasedEdgeColumnFamilies.java
@@ -30,9 +30,8 @@ import org.apache.cassandra.db.marshal.DynamicCompositeType;
import org.apache.usergrid.persistence.core.astyanax.ColumnTypes;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
-import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdge;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeColumnFamilies;
import org.apache.usergrid.persistence.graph.serialization.impl.shard.EdgeRowKey;
@@ -78,17 +77,17 @@ public class SizebasedEdgeColumnFamilies implements EdgeColumnFamilies {
//initialize the CF's from our implementation
private static final MultiTennantColumnFamily<ScopedRowKey<RowKey>, DirectedEdge> SOURCE_NODE_EDGES =
new MultiTennantColumnFamily<>( "Graph_Source_Node_Edges",
- new OrganizationScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
+ new ScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
private static final MultiTennantColumnFamily<ScopedRowKey<RowKey>, DirectedEdge> TARGET_NODE_EDGES =
new MultiTennantColumnFamily<>( "Graph_Target_Node_Edges",
- new OrganizationScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
+ new ScopedRowKeySerializer<>( ROW_SERIALIZER ), EDGE_SERIALIZER );
private static final MultiTennantColumnFamily<ScopedRowKey<RowKeyType>, DirectedEdge> SOURCE_NODE_TARGET_TYPE =
new MultiTennantColumnFamily<>( "Graph_Source_Node_Target_Type",
- new OrganizationScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
+ new ScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
/**
@@ -96,12 +95,12 @@ public class SizebasedEdgeColumnFamilies implements EdgeColumnFamilies {
*/
private static final MultiTennantColumnFamily<ScopedRowKey<RowKeyType>, DirectedEdge> TARGET_NODE_SOURCE_TYPE =
new MultiTennantColumnFamily<>( "Graph_Target_Node_Source_Type",
- new OrganizationScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
+ new ScopedRowKeySerializer<>( ROW_TYPE_SERIALIZER ), EDGE_SERIALIZER );
private static final MultiTennantColumnFamily<ScopedRowKey<EdgeRowKey>, Long> EDGE_VERSIONS =
new MultiTennantColumnFamily<>( "Graph_Edge_Versions",
- new OrganizationScopedRowKeySerializer<>( EDGE_ROW_KEY_SERIALIZER ), LONG_SERIALIZER );
+ new ScopedRowKeySerializer<>( EDGE_ROW_KEY_SERIALIZER ), LONG_SERIALIZER );
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6b203de0/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
index f43a500..7f2598b 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
@@ -28,15 +28,14 @@ import com.google.common.base.Preconditions;
import org.apache.cassandra.db.marshal.BytesType;
import org.apache.cassandra.db.marshal.UTF8Type;
-import org.apache.usergrid.persistence.core.astyanax.BucketLocator;
import org.apache.usergrid.persistence.core.astyanax.CompositeFieldSerializer;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamily;
import org.apache.usergrid.persistence.core.astyanax.MultiTennantColumnFamilyDefinition;
import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKey;
-import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedBucketRowKeySerializer;
-import org.apache.usergrid.persistence.core.astyanax.OrganizationScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.BucketScopedRowKeySerializer;
+import org.apache.usergrid.persistence.core.astyanax.ScopedRowKeySerializer;
import org.apache.usergrid.persistence.core.astyanax.ScopedRowKey;
-import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.hash.ExpandingBucketLocator;
import org.apache.usergrid.persistence.map.MapScope;
import com.google.common.hash.Funnel;
@@ -59,13 +58,13 @@ public class MapSerializationImpl implements MapSerialization {
private static final MapKeySerializer KEY_SERIALIZER = new MapKeySerializer();
- private static final OrganizationScopedBucketRowKeySerializer<String> MAP_KEY_SERIALIZER =
- new OrganizationScopedBucketRowKeySerializer<>( KEY_SERIALIZER );
+ private static final BucketScopedRowKeySerializer<String> MAP_KEY_SERIALIZER =
+ new BucketScopedRowKeySerializer<>( KEY_SERIALIZER );
private static final MapEntrySerializer ENTRY_SERIALIZER = new MapEntrySerializer();
- private static final OrganizationScopedRowKeySerializer<MapEntryKey> MAP_ENTRY_SERIALIZER =
- new OrganizationScopedRowKeySerializer<>( ENTRY_SERIALIZER );
+ private static final ScopedRowKeySerializer<MapEntryKey> MAP_ENTRY_SERIALIZER =
+ new ScopedRowKeySerializer<>( ENTRY_SERIALIZER );
private static final BooleanSerializer BOOLEAN_SERIALIZER = BooleanSerializer.get();
@@ -88,9 +87,9 @@ public class MapSerializationImpl implements MapSerialization {
new MultiTennantColumnFamily<>( "Map_Keys", MAP_KEY_SERIALIZER, STRING_SERIALIZER );
/**
- * Number of buckets to hash across
+ * Number of buckets to hash across.
*/
- private static final int NUM_BUCKETS = 20;
+ private static final int[] NUM_BUCKETS = {20};
/**
* How to funnel keys for buckets
@@ -108,7 +107,7 @@ public class MapSerializationImpl implements MapSerialization {
/**
* Locator to get us all buckets
*/
- private static final BucketLocator<String> BUCKET_LOCATOR = new BucketLocator<>( NUM_BUCKETS, MAP_KEY_FUNNEL );
+ private static final ExpandingBucketLocator<String> BUCKET_LOCATOR = new ExpandingBucketLocator<>(MAP_KEY_FUNNEL, NUM_BUCKETS);
private final Keyspace keyspace;
@@ -140,7 +139,7 @@ public class MapSerializationImpl implements MapSerialization {
//add it to the keys
- final int bucket = BUCKET_LOCATOR.getBucket( key );
+ final int bucket = BUCKET_LOCATOR.getCurrentBucket( key );
final BucketScopedRowKey< String> keyRowKey =
BucketScopedRowKey.fromKey( scope.getApplication(), key, bucket);
@@ -179,7 +178,7 @@ public class MapSerializationImpl implements MapSerialization {
//add it to the keys
- final int bucket = BUCKET_LOCATOR.getBucket( key );
+ final int bucket = BUCKET_LOCATOR.getCurrentBucket( key );
final BucketScopedRowKey< String> keyRowKey =
BucketScopedRowKey.fromKey( scope.getApplication(), key, bucket);
@@ -217,7 +216,7 @@ public class MapSerializationImpl implements MapSerialization {
batch.withRow(MAP_ENTRIES, entryRowKey).putColumn(true, value);
//add it to the keys
- final int bucket = BUCKET_LOCATOR.getBucket( key );
+ final int bucket = BUCKET_LOCATOR.getCurrentBucket( key );
final BucketScopedRowKey< String> keyRowKey =
BucketScopedRowKey.fromKey( scope.getApplication(), key, bucket);
@@ -237,13 +236,16 @@ public class MapSerializationImpl implements MapSerialization {
//serialize to the entry
batch.withRow(MAP_ENTRIES, entryRowKey).delete();
- //add it to the keys
+ //add it to the keys, we're not sure which one it may have come from
+ final int[] buckets = BUCKET_LOCATOR.getAllBuckets( key );
- final int bucket = BUCKET_LOCATOR.getBucket( key );
- final BucketScopedRowKey<String> rowKey = BucketScopedRowKey.fromKey( scope.getApplication(), key, bucket );
+ final List<BucketScopedRowKey<String>>
+ rowKeys = BucketScopedRowKey.fromRange( scope.getApplication(), key, buckets );
- batch.withRow( MAP_KEYS, rowKey ).deleteColumn( key );
+ for(BucketScopedRowKey<String> rowKey: rowKeys) {
+ batch.withRow( MAP_KEYS, rowKey ).deleteColumn( key );
+ }
executeBatch( batch );
}
@@ -272,9 +274,12 @@ public class MapSerializationImpl implements MapSerialization {
private Column<Boolean> getValue(MapScope scope, String key) {
+
+
//add it to the entry
final ScopedRowKey<MapEntryKey> entryRowKey = MapEntryKey.fromKey(scope, key);
+ //now get all columns, including the "old row key value"
try {
final Column<Boolean> result = keyspace.prepareQuery( MAP_ENTRIES )
.getKey( entryRowKey ).getColumn( true ).execute().getResult();