You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2016/08/22 15:00:47 UTC
[43/50] usergrid git commit: Fix issue with map manager key sorting.
Enhance buffer sizing by using byte length of string components.
Fix issue with map manager key sorting. Enhance buffer sizing by using byte length of string components.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fcbe8036
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fcbe8036
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fcbe8036
Branch: refs/heads/asf-site
Commit: fcbe803679ab4832a9ec2b3d26fd35eaec61202d
Parents: 92d5f40
Author: Michael Russo <mr...@apigee.com>
Authored: Thu Aug 18 23:55:30 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Thu Aug 18 23:55:30 2016 -0700
----------------------------------------------------------------------
.../impl/ScopedCacheSerializationImpl.java | 2 +-
.../UniqueValueSerializationStrategyV1Impl.java | 23 ++++++----
.../UniqueValueSerializationStrategyV2Impl.java | 20 ++++----
.../usergrid/persistence/map/MapManager.java | 2 +-
.../map/impl/MapSerializationImpl.java | 4 +-
.../persistence/map/MapManagerTest.java | 48 +++++++++++++++++---
6 files changed, 70 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
index 1334650..2a44f2b 100644
--- a/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
+++ b/stack/corepersistence/cache/src/main/java/org/apache/usergrid/persistence/cache/impl/ScopedCacheSerializationImpl.java
@@ -303,7 +303,7 @@ public class ScopedCacheSerializationImpl<K,V> implements ScopedCacheSerializati
keys.add(3, rowKeyString);
// UUIDs are 16 bytes, allocate the buffer accordingly
- int size = 16+ownerType.length()+rowKeyString.length();
+ int size = 16+ownerType.getBytes().length+rowKeyString.getBytes().length;
// ints are 4 bytes, add for the bucket
size += 4;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
index 55ba011..1435c1e 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV1Impl.java
@@ -198,14 +198,15 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ
String comparator = UUID_TYPE_REVERSED;
- int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+
- fieldEntry.getField().getTypeName().name().length();
+ int size = 16+fieldEntry.getField().getName().getBytes().length
+ +fieldEntry.getField().getValue().toString().getBytes().length+
+ fieldEntry.getField().getTypeName().name().getBytes().length;
// we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality
size += keys.size()*5;
// uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow
- size += keys.size()*comparator.length();
+ size += keys.size()*comparator.getBytes().length;
ByteBuffer stuff = ByteBuffer.allocate(size);
@@ -228,7 +229,7 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ
stuff.putShort((short) ('\u8000' | a));
}else{
comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here
- stuff.putShort((short)comparator.length());
+ stuff.putShort((short)comparator.getBytes().length);
stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
}
@@ -293,13 +294,13 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ
keys.add(entityVersion.getEntityId().getType());
// UUIDs are 16 bytes
- int size = 16+16+entityVersion.getEntityId().getType().length();
+ int size = 16+16+entityVersion.getEntityId().getType().getBytes().length;
// we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality
size += keys.size()*5;
// we always add comparator to the buffer as well
- size += keys.size()*comparator.length();
+ size += keys.size()*comparator.getBytes().length;
ByteBuffer stuff = ByteBuffer.allocate(size);
@@ -311,7 +312,7 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ
comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text
}
- stuff.putShort((short)comparator.length());
+ stuff.putShort((short)comparator.getBytes().length);
stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED);
@@ -457,8 +458,9 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ
// UUIDs are 16 bytes, allocate the buffer accordingly
- int size = 16 + applicationType.length() + 16 + applicationType.length() + collectionName.length() +
- fieldType.length() + fieldName.length()+fieldValueString.length();
+ int size = 16 + applicationType.getBytes().length + 16 + applicationType.getBytes().length +
+ collectionName.getBytes().length + fieldType.getBytes().length + fieldName.getBytes().length
+ + fieldValueString.getBytes().length;
// we always need to add length for the 2 byte short and 1 byte equality
@@ -503,7 +505,8 @@ public class UniqueValueSerializationStrategyV1Impl extends UniqueValueSerializ
keys.add(entityId);
keys.add(entityType);
- int size = 16+applicationType.length()+16+applicationType.length()+collectionName.length()+16+entityType.length();
+ int size = 16+applicationType.getBytes().length+16+applicationType.getBytes().length
+ +collectionName.getBytes().length+16+entityType.getBytes().length;
// we always need to add length for the 2 byte short and 1 byte equality
size += keys.size()*3;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
index a5fceeb..518937d 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/serialization/impl/UniqueValueSerializationStrategyV2Impl.java
@@ -187,14 +187,15 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ
String comparator = UUID_TYPE_REVERSED;
- int size = 16+fieldEntry.getField().getName().length()+fieldEntry.getField().getValue().toString().length()+
- fieldEntry.getField().getTypeName().name().length();
+ int size = 16+fieldEntry.getField().getName().getBytes().length
+ +fieldEntry.getField().getValue().toString().getBytes().length+
+ fieldEntry.getField().getTypeName().name().getBytes().length;
// we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality
size += keys.size()*5;
// uuid type comparator is longest, ensure we allocate buffer using the max size to avoid overflow
- size += keys.size()*comparator.length();
+ size += keys.size()*comparator.getBytes().length;
ByteBuffer stuff = ByteBuffer.allocate(size);
@@ -217,7 +218,7 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ
stuff.putShort((short) ('\u8000' | a));
}else{
comparator = "UTF8Type"; // only strings are being serialized other than UUIDs here
- stuff.putShort((short)comparator.length());
+ stuff.putShort((short)comparator.getBytes().length);
stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
}
@@ -282,13 +283,13 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ
keys.add(entityVersion.getEntityId().getType());
// UUIDs are 16 bytes
- int size = 16+16+entityVersion.getEntityId().getType().length();
+ int size = 16+16+entityVersion.getEntityId().getType().getBytes().length;
// we always need to add length for the 2 byte comparator short, 2 byte length short and 1 byte equality
size += keys.size()*5;
// we always add comparator to the buffer as well
- size += keys.size()*comparator.length();
+ size += keys.size()*comparator.getBytes().length;
ByteBuffer stuff = ByteBuffer.allocate(size);
@@ -303,7 +304,7 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ
comparator = "UTF8Type"; // if it's not a UUID, the only other thing we're serializing is text
}
- stuff.putShort((short)comparator.length());
+ stuff.putShort((short)comparator.getBytes().length);
stuff.put(DataType.serializeValue(comparator, ProtocolVersion.NEWEST_SUPPORTED));
ByteBuffer kb = DataType.serializeValue(key, ProtocolVersion.NEWEST_SUPPORTED);
@@ -436,7 +437,8 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ
// UUIDs are 16 bytes, allocate the buffer accordingly
- int size = 16 + applicationType.length() + entityType.length() + fieldType.length() + fieldName.length()+fieldValueString.length();
+ int size = 16 + applicationType.getBytes().length + entityType.getBytes().length
+ + fieldType.getBytes().length + fieldName.getBytes().length+fieldValueString.getBytes().length;
// we always need to add length for the 2 byte short and 1 byte equality
@@ -470,7 +472,7 @@ public class UniqueValueSerializationStrategyV2Impl extends UniqueValueSerializ
keys.add(entityId);
keys.add(entityType);
- int size = 16+applicationType.length()+16+entityType.length();
+ int size = 16+applicationType.getBytes().length+16+entityType.getBytes().length;
// we always need to add length for the 2 byte short and 1 byte equality
size += keys.size()*3;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
index e1d121e..bd1f444 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/MapManager.java
@@ -92,7 +92,7 @@ public interface MapManager {
void delete( final String key );
/**
- * Return a page of keys that exist within the map. Keys are sorted descending.
+ * Return a page of keys that exist within the map. Keys are sorted ascending in lexicographical ordering.
* @param cursor
* @param limit
*/
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
index 282974c..2c77ebf64 100644
--- a/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
+++ b/stack/corepersistence/map/src/main/java/org/apache/usergrid/persistence/map/impl/MapSerializationImpl.java
@@ -70,7 +70,7 @@ public class MapSerializationImpl implements MapSerialization {
put( "column1", DataType.Name.BLOB );
put( "value", DataType.Name.BLOB ); }};
private static final Map<String, String> MAP_KEYS_CLUSTERING_ORDER =
- new HashMap<String, String>(){{ put( "column1", "DESC" ); }};
+ new HashMap<String, String>(){{ put( "column1", "ASC" ); }};
@@ -465,7 +465,7 @@ public class MapSerializationImpl implements MapSerialization {
}
// UUIDs are 16 bytes, allocate the buffer accordingly
- int size = 16+ownerType.length()+mapName.length()+mapKey.length();
+ int size = 16+ownerType.getBytes().length+mapName.getBytes().length+mapKey.getBytes().length;
if(bucketNumber > 0 ){
// ints are 4 bytes
size += 4;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcbe8036/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
index 89f6799..07bec81 100644
--- a/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
+++ b/stack/corepersistence/map/src/test/java/org/apache/usergrid/persistence/map/MapManagerTest.java
@@ -41,6 +41,7 @@ import com.google.inject.Inject;
import static junit.framework.TestCase.assertNotNull;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
@RunWith( ITRunner.class )
@@ -121,20 +122,20 @@ public class MapManagerTest {
MapKeyResults keyResults = mm.getKeys(null, 3);
assertEquals(3, keyResults.getKeys().size());
- assertEquals(key6, keyResults.getKeys().get(0));
+ assertTrue("should contain key1", keyResults.getKeys().contains(key1));
assertNotNull(keyResults.getCursor());
MapKeyResults keyResults2 = mm.getKeys(keyResults.getCursor(), 3);
assertEquals(3, keyResults2.getKeys().size());
- assertEquals(key3, keyResults2.getKeys().get(0));
+ assertTrue("should contain key4", keyResults2.getKeys().contains(key4));
}
@Test
- public void getAllKeysAfterDelete(){
+ public void testKeysOrdering(){
MapManager mm = mmf.createMapManager(this.scope);
@@ -157,8 +158,45 @@ public class MapManagerTest {
MapKeyResults keyResults = mm.getKeys(null, 6);
assertEquals(6, keyResults.getKeys().size());
- assertEquals(key6, keyResults.getKeys().get(0));
+ assertEquals(key1, keyResults.getKeys().get(0));
+
+
+ mm.delete(key1);
+ mm.delete(key2);
+ mm.delete(key3);
+
+ MapKeyResults keyResults2 = mm.getKeys(null, 6);
+
+ assertEquals(3, keyResults2.getKeys().size());
+ assertEquals(key4, keyResults2.getKeys().get(0));
+
+
+ }
+ @Test
+ public void getAllKeysAfterDelete(){
+
+ MapManager mm = mmf.createMapManager(this.scope);
+
+ final String value = "value";
+
+ final String key1 = "key1";
+ final String key2 = "key2";
+ final String key3 = "key3";
+ final String key4 = "key4";
+ final String key5 = "key5";
+ final String key6 = "key6";
+
+ mm.putString( key1, value );
+ mm.putString( key2, value );
+ mm.putString( key3, value );
+ mm.putString( key4, value );
+ mm.putString( key5, value );
+ mm.putString( key6, value );
+
+ MapKeyResults keyResults = mm.getKeys(null, 6);
+
+ assertEquals(6, keyResults.getKeys().size());
mm.delete(key4);
mm.delete(key5);
@@ -167,8 +205,6 @@ public class MapManagerTest {
MapKeyResults keyResults2 = mm.getKeys(null, 6);
assertEquals(3, keyResults2.getKeys().size());
- assertEquals(key3, keyResults2.getKeys().get(0));
-
}