You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/11/19 22:11:34 UTC
[2/2] incubator-usergrid git commit: Added new field buffer for
storing large data elements in Cassandra
Added new field buffer for storing large data elements in Cassandra
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/39b15765
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/39b15765
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/39b15765
Branch: refs/heads/USERGRID-250-buffer-size-fix
Commit: 39b157651b1316d7e1c76e592f5a2774a775f7f6
Parents: 3073fe8
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 19 14:11:30 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 19 14:11:30 2014 -0700
----------------------------------------------------------------------
.../persistence/core/astyanax/FieldBuffer.java | 70 ++++++++++
.../core/astyanax/FieldBufferBuilder.java | 87 ++++++++++++
.../core/astyanax/FieldBufferParser.java | 81 +++++++++++
.../core/astyanax/FieldBufferSerializer.java | 134 +++++++++++++++++++
.../astyanax/FieldBufferSerializerTest.java | 126 +++++++++++++++++
5 files changed, 498 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
new file mode 100644
index 0000000..898bc73
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+
+/**
+ * A utility class for storing multiple fields
+ */
+public class FieldBuffer {
+
+ private final List<ByteBuffer> fields;
+
+
+ /**
+ * Allocate a new field buffer with the expected max size. This allows us to pre-allocate
+ * our buffer for fields
+ *
+ * @param expectedMax
+ */
+ public FieldBuffer(final int expectedMax){
+ fields = new ArrayList<>( expectedMax );
+ }
+
+
+ /**
+ * Add the field and the serializer to the list
+ * @param value The serialized value to add to the buffer
+ */
+ public void add(ByteBuffer value){
+ //Note that we're not validating our byte buffer length. Since the length of a byte buffer IS an integer
+ //we can't possibly overflow an integer because they're the same data type
+ fields.add(value.duplicate() );
+ }
+
+
+ /**
+ * Return the list of all fields in read only format
+ * @return
+ */
+ public List<ByteBuffer> getFields(){
+ return Collections.unmodifiableList(fields);
+ }
+
+
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
new file mode 100644
index 0000000..88cfb62
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import com.netflix.astyanax.serializers.ByteBufferSerializer;
+import com.netflix.astyanax.serializers.ByteSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+
+
+/**
+ * A builder pattern
+ */
+public class FieldBufferBuilder {
+
+
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+ private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+ private static final ByteSerializer BYTE_SERIALIZER = ByteSerializer.get();
+
+ private final FieldBuffer buffer;
+
+
+ public FieldBufferBuilder( final int elementSize ) {
+ buffer = new FieldBuffer( elementSize );
+ }
+
+
+ /**
+ * Add the integer to the fields.
+ * @param value
+ * @return
+ */
+ public FieldBufferBuilder addInteger( final int value ) {
+ buffer.add( INTEGER_SERIALIZER.toByteBuffer( value ) );
+ return this;
+ }
+
+
+ /**
+ * Add the byte array to the fieldbuilder
+ * @param bytes
+ * @return
+ */
+ public FieldBufferBuilder addBytes( final byte[] bytes ) {
+ buffer.add( BYTES_ARRAY_SERIALIZER.toByteBuffer( bytes ) );
+ return this;
+ }
+
+
+ /**
+ * Add the bytes to the fieldBuilder
+ * @param newByte
+ * @return
+ */
+ public FieldBufferBuilder addByte( final byte newByte ) {
+ buffer.add( BYTE_SERIALIZER.toByteBuffer( newByte ) );
+ return this;
+ }
+
+
+ /**
+ * Return the field buffer from the builder
+ * @return
+ */
+ public FieldBuffer build(){
+ return buffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
new file mode 100644
index 0000000..ef3b962
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import com.netflix.astyanax.serializers.ByteSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+
+
+/**
+ * A parser for our field buffer
+ */
+public class FieldBufferParser {
+
+
+ private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+ private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+ private static final ByteSerializer BYTE_SERIALIZER = ByteSerializer.get();
+
+ private final Iterator<ByteBuffer> fields;
+
+
+ public FieldBufferParser( final FieldBuffer buffer ) {
+ fields = buffer.getFields().iterator();
+ }
+
+
+ /**
+ * Return the value as an integer
+ */
+ public int getInteger() {
+ return INTEGER_SERIALIZER.fromByteBuffer( getNext() );
+ }
+
+
+ /**
+ * Return the value as a byte array
+ */
+ public byte[] getBytes() {
+ return BYTES_ARRAY_SERIALIZER.fromByteBuffer( getNext() );
+ }
+
+
+ /**
+ * return the next vlaue as a byte
+ */
+ public byte getByte() {
+ return BYTE_SERIALIZER.fromByteBuffer( getNext() );
+ }
+
+
+ private ByteBuffer getNext() {
+ if ( !fields.hasNext() ) {
+ throw new NoSuchElementException( "No more elements to return" );
+ }
+
+ return fields.next();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
new file mode 100644
index 0000000..d865349
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import com.netflix.astyanax.connectionpool.exceptions.SerializationException;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+
+
+/**
+ * Serializer that tests serializing field buggers
+ */
+public class FieldBufferSerializer extends AbstractSerializer<FieldBuffer> {
+
+
+ private static final FieldBufferSerializer INSTANCE = new FieldBufferSerializer();
+
+ private static final byte VERSION = 0;
+
+
+ public static FieldBufferSerializer get() {
+ return INSTANCE;
+ }
+
+
+ @Override
+ public ByteBuffer toByteBuffer( final FieldBuffer obj ) {
+ //create an output stream
+ final List<ByteBuffer> fields = obj.getFields();
+ final int fieldCount = fields.size();
+
+ //we start with 9. 1 byte for version, 4 bytes for total length, 4 bytes for the field count
+ int totalLength = 9;
+
+ for ( ByteBuffer fieldData : fields ) {
+ final int bufferLength = fieldData.remaining();
+ totalLength += 4 + bufferLength;
+ }
+
+ //now we have our total length, allocate it.
+
+ ByteBuffer buffer = ByteBuffer.allocate( totalLength );
+
+ buffer.put( VERSION );
+ buffer.putInt( totalLength );
+ buffer.putInt( fieldCount );
+
+ for ( ByteBuffer fieldData : fields ) {
+ final int bufferLength = fieldData.limit();
+
+ buffer.putInt( bufferLength );
+ buffer.put( fieldData );
+ }
+
+ buffer.rewind();
+ return buffer;
+ }
+
+
+ @Override
+ public FieldBuffer fromByteBuffer( final ByteBuffer byteBuffer ) {
+
+ final int totalSize = byteBuffer.remaining();
+ final byte version = byteBuffer.get();
+
+
+ //not what we expected, throw an ex
+ if ( version != VERSION ) {
+ throw new SerializationException(
+ "Unable to de-serialze. Expected version " + VERSION + " but was version " + version );
+ }
+
+ final int expectedTotalSize = byteBuffer.getInt();
+
+ if ( totalSize != expectedTotalSize ) {
+ throw new SerializationException(
+ "The total size we expected was different. Stored total size is " + expectedTotalSize
+ + " but actual buffer size is " + totalSize );
+ }
+
+
+ final int numberFields = byteBuffer.getInt();
+
+ final FieldBuffer buffer = new FieldBuffer( numberFields );
+
+
+ for ( int i = 0; i < numberFields; i++ ) {
+ final int bufferLength = byteBuffer.getInt();
+
+ final int startPosition = byteBuffer.position();
+ final int newLimit = startPosition + bufferLength;
+
+ //now read in the length and add it to our fieldBuffer
+
+
+ //we do this so we don't actually copy the underlying buff again. Instead we duplicate it
+ //and change our start and limits, to ensure we only read the field data when parsing
+ final ByteBuffer fieldData = byteBuffer.duplicate();
+
+ //set the limit that's the end of the field on the duplicate so that we won't read beyond this value
+ fieldData.limit( newLimit );
+
+ //advance our own internal buffer so that we can read the next field
+ byteBuffer.position(newLimit);
+
+
+
+ buffer.add( fieldData );
+ }
+
+
+ return buffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
new file mode 100644
index 0000000..e9a8194
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+
+/**
+ * Tests the builder, parser, and serialization of the entities
+ */
+public class FieldBufferSerializerTest {
+
+ /**
+ * Perform a very simple serialization of the 3 field types
+ */
+ @Test
+ public void simpleSerializer() {
+
+ final byte setByte = 1;
+ final int setInteger = 200;
+ final byte[] setByteArray = generateByteArray( 1000 );
+
+ FieldBufferBuilder builder = new FieldBufferBuilder( 3 );
+
+ builder.addByte( setByte );
+ builder.addInteger( setInteger );
+ builder.addBytes( setByteArray );
+
+
+ final FieldBufferSerializer serializer = new FieldBufferSerializer();
+
+ final ByteBuffer serialized = serializer.toByteBuffer( builder.build() );
+
+
+ final FieldBuffer parsed = serializer.fromByteBuffer( serialized );
+
+ FieldBufferParser parser = new FieldBufferParser( parsed );
+
+ final byte returnedByte = parser.getByte();
+
+ assertEquals( "Bytes should be equal", setByte, returnedByte );
+
+ final int returnedInt = parser.getInteger();
+
+ assertEquals( "Integer should be equal", setInteger, returnedInt );
+
+ final byte[] returnedByteArray = parser.getBytes();
+
+ assertArrayEquals( "arrays should be equal", setByteArray, returnedByteArray );
+ }
+
+
+ @Test
+ public void largerThanUnsignedShorts() {
+ final int maxShortSize = 65535;
+
+ final int lengthOfArray = maxShortSize + 1;
+
+ final byte setByte = 2;
+ final int setInteger = 400;
+
+
+ final byte[] setByteArray = generateByteArray( lengthOfArray );
+
+ FieldBufferBuilder builder = new FieldBufferBuilder( 1 );
+
+ builder.addBytes( setByteArray );
+ builder.addByte( setByte );
+ builder.addInteger( setInteger );
+
+ final FieldBufferSerializer serializer = new FieldBufferSerializer();
+
+ final ByteBuffer serialized = serializer.toByteBuffer( builder.build() );
+
+
+ final FieldBuffer parsed = serializer.fromByteBuffer( serialized );
+
+ FieldBufferParser parser = new FieldBufferParser( parsed );
+
+ final byte[] returnedArray = parser.getBytes();
+
+ assertArrayEquals( setByteArray, returnedArray );
+
+ final byte returnedByte = parser.getByte();
+
+ assertEquals( "Bytes should be equal", setByte, returnedByte );
+
+ final int returnedInt = parser.getInteger();
+
+ assertEquals( "Integer should be equal", setInteger, returnedInt );
+ }
+
+
+ private byte[] generateByteArray( final int length ) {
+ final byte[] data = new byte[length];
+
+ //set it to something other than 0 so we can be sure we're allocating correctly on parse
+ Arrays.fill( data, ( byte ) 1 );
+
+ return data;
+ }
+}