You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/11/19 22:11:34 UTC

[2/2] incubator-usergrid git commit: Added new field buffer for storing large data elements in Cassandra

Added new field buffer for storing large data elements in Cassandra


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/39b15765
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/39b15765
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/39b15765

Branch: refs/heads/USERGRID-250-buffer-size-fix
Commit: 39b157651b1316d7e1c76e592f5a2774a775f7f6
Parents: 3073fe8
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 19 14:11:30 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 19 14:11:30 2014 -0700

----------------------------------------------------------------------
 .../persistence/core/astyanax/FieldBuffer.java  |  70 ++++++++++
 .../core/astyanax/FieldBufferBuilder.java       |  87 ++++++++++++
 .../core/astyanax/FieldBufferParser.java        |  81 +++++++++++
 .../core/astyanax/FieldBufferSerializer.java    | 134 +++++++++++++++++++
 .../astyanax/FieldBufferSerializerTest.java     | 126 +++++++++++++++++
 5 files changed, 498 insertions(+)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
new file mode 100644
index 0000000..898bc73
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBuffer.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+
+/**
+ * A utility class for storing multiple fields
+ */
+public class FieldBuffer {
+
+    private final List<ByteBuffer> fields;
+
+
+    /**
+     * Allocate a new field buffer with the expected max size.  This allows us to pre-allocate
+     * our buffer for fields
+     *
+     * @param expectedMax
+     */
+    public FieldBuffer(final int expectedMax){
+        fields = new ArrayList<>( expectedMax );
+    }
+
+
+    /**
+     * Add the field and the serializer to the list
+     * @param value   The serialized value to add to the buffer
+     */
+    public void add(ByteBuffer value){
+        //Note that we're not validating our byte buffer length.  Since the length of a byte buffer IS an integer
+        //we can't possibly overflow an integer because they're the same data type
+        fields.add(value.duplicate() );
+    }
+
+
+    /**
+     * Return the list of all fields in read only format
+     * @return
+     */
+    public List<ByteBuffer> getFields(){
+        return Collections.unmodifiableList(fields);
+    }
+
+
+
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
new file mode 100644
index 0000000..88cfb62
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferBuilder.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import com.netflix.astyanax.serializers.ByteBufferSerializer;
+import com.netflix.astyanax.serializers.ByteSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+
+
+/**
+ * A builder pattern
+ */
+public class FieldBufferBuilder {
+
+
+    private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+    private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+    private static final ByteSerializer BYTE_SERIALIZER = ByteSerializer.get();
+
+    private final FieldBuffer buffer;
+
+
+    public FieldBufferBuilder( final int elementSize ) {
+        buffer = new FieldBuffer( elementSize );
+    }
+
+
+    /**
+     * Add the integer to the fields.
+     * @param value
+     * @return
+     */
+    public FieldBufferBuilder addInteger( final int value ) {
+        buffer.add( INTEGER_SERIALIZER.toByteBuffer( value ) );
+        return this;
+    }
+
+
+    /**
+     * Add the byte array to the fieldbuilder
+     * @param bytes
+     * @return
+     */
+    public FieldBufferBuilder addBytes( final byte[] bytes ) {
+        buffer.add( BYTES_ARRAY_SERIALIZER.toByteBuffer( bytes ) );
+        return this;
+    }
+
+
+    /**
+     * Add the bytes to the fieldBuilder
+     * @param newByte
+     * @return
+     */
+    public FieldBufferBuilder addByte( final byte newByte ) {
+        buffer.add( BYTE_SERIALIZER.toByteBuffer( newByte ) );
+        return this;
+    }
+
+
+    /**
+     * Return the field buffer from the builder
+     * @return
+     */
+    public FieldBuffer build(){
+        return buffer;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
new file mode 100644
index 0000000..ef3b962
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferParser.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import com.netflix.astyanax.serializers.ByteSerializer;
+import com.netflix.astyanax.serializers.BytesArraySerializer;
+import com.netflix.astyanax.serializers.IntegerSerializer;
+
+
+/**
+ * A parser for our field buffer
+ */
+public class FieldBufferParser {
+
+
+    private static final IntegerSerializer INTEGER_SERIALIZER = IntegerSerializer.get();
+    private static final BytesArraySerializer BYTES_ARRAY_SERIALIZER = BytesArraySerializer.get();
+    private static final ByteSerializer BYTE_SERIALIZER = ByteSerializer.get();
+
+    private final Iterator<ByteBuffer> fields;
+
+
+    public FieldBufferParser( final FieldBuffer buffer ) {
+        fields = buffer.getFields().iterator();
+    }
+
+
+    /**
+     * Return the value as an integer
+     */
+    public int getInteger() {
+        return INTEGER_SERIALIZER.fromByteBuffer( getNext() );
+    }
+
+
+    /**
+     * Return the value as a byte array
+     */
+    public byte[] getBytes() {
+        return BYTES_ARRAY_SERIALIZER.fromByteBuffer( getNext() );
+    }
+
+
+    /**
+     * return the next vlaue as a byte
+     */
+    public byte getByte() {
+        return BYTE_SERIALIZER.fromByteBuffer( getNext() );
+    }
+
+
+    private ByteBuffer getNext() {
+        if ( !fields.hasNext() ) {
+            throw new NoSuchElementException( "No more elements to return" );
+        }
+
+        return fields.next();
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
new file mode 100644
index 0000000..d865349
--- /dev/null
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializer.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import com.netflix.astyanax.connectionpool.exceptions.SerializationException;
+import com.netflix.astyanax.serializers.AbstractSerializer;
+
+
+/**
+ * Serializer that tests serializing field buggers
+ */
+public class FieldBufferSerializer extends AbstractSerializer<FieldBuffer> {
+
+
+    private static final FieldBufferSerializer INSTANCE = new FieldBufferSerializer();
+
+    private static final byte VERSION = 0;
+
+
+    public static FieldBufferSerializer get() {
+        return INSTANCE;
+    }
+
+
+    @Override
+    public ByteBuffer toByteBuffer( final FieldBuffer obj ) {
+        //create an output stream
+        final List<ByteBuffer> fields = obj.getFields();
+        final int fieldCount = fields.size();
+
+        //we start with 9.  1 byte for version, 4 bytes for total length, 4 bytes for the field count
+        int totalLength = 9;
+
+        for ( ByteBuffer fieldData : fields ) {
+            final int bufferLength = fieldData.remaining();
+            totalLength += 4 + bufferLength;
+        }
+
+        //now we have our total length, allocate it.
+
+        ByteBuffer buffer = ByteBuffer.allocate( totalLength );
+
+        buffer.put( VERSION );
+        buffer.putInt( totalLength );
+        buffer.putInt( fieldCount );
+
+        for ( ByteBuffer fieldData : fields ) {
+            final int bufferLength = fieldData.limit();
+
+            buffer.putInt( bufferLength );
+            buffer.put( fieldData );
+        }
+
+        buffer.rewind();
+        return buffer;
+    }
+
+
+    @Override
+    public FieldBuffer fromByteBuffer( final ByteBuffer byteBuffer ) {
+
+        final int totalSize = byteBuffer.remaining();
+        final byte version = byteBuffer.get();
+
+
+        //not what we expected, throw an ex
+        if ( version != VERSION ) {
+            throw new SerializationException(
+                    "Unable to de-serialze. Expected version " + VERSION + " but was version " + version );
+        }
+
+        final int expectedTotalSize = byteBuffer.getInt();
+
+        if ( totalSize != expectedTotalSize ) {
+            throw new SerializationException(
+                    "The total size we expected was different.  Stored total size is " + expectedTotalSize
+                            + " but actual buffer size is " + totalSize );
+        }
+
+
+        final int numberFields = byteBuffer.getInt();
+
+        final FieldBuffer buffer = new FieldBuffer( numberFields );
+
+
+        for ( int i = 0; i < numberFields; i++ ) {
+            final int bufferLength = byteBuffer.getInt();
+
+            final int startPosition = byteBuffer.position();
+            final int newLimit = startPosition + bufferLength;
+
+            //now read in the length and add it to our fieldBuffer
+
+
+            //we do this so we don't actually copy the underlying buff again. Instead we duplicate it
+            //and change our start and limits, to ensure we only read the field data when parsing
+            final ByteBuffer fieldData = byteBuffer.duplicate();
+
+            //set the limit that's the end of the field on the duplicate so that we won't read beyond this value
+            fieldData.limit( newLimit );
+
+            //advance our own internal buffer so that we can read the next field
+            byteBuffer.position(newLimit);
+
+
+
+            buffer.add( fieldData );
+        }
+
+
+        return buffer;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/39b15765/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
new file mode 100644
index 0000000..e9a8194
--- /dev/null
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/FieldBufferSerializerTest.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.persistence.core.astyanax;
+
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
+
+/**
+ * Tests the builder, parser, and serialization of the entities
+ */
+public class FieldBufferSerializerTest {
+
+    /**
+     * Perform a very simple serialization of the 3 field types
+     */
+    @Test
+    public void simpleSerializer() {
+
+        final byte setByte = 1;
+        final int setInteger = 200;
+        final byte[] setByteArray = generateByteArray( 1000 );
+
+        FieldBufferBuilder builder = new FieldBufferBuilder( 3 );
+
+        builder.addByte( setByte );
+        builder.addInteger( setInteger );
+        builder.addBytes( setByteArray );
+
+
+        final FieldBufferSerializer serializer = new FieldBufferSerializer();
+
+        final ByteBuffer serialized = serializer.toByteBuffer( builder.build() );
+
+
+        final FieldBuffer parsed = serializer.fromByteBuffer( serialized );
+
+        FieldBufferParser parser = new FieldBufferParser( parsed );
+
+        final byte returnedByte = parser.getByte();
+
+        assertEquals( "Bytes should be equal", setByte, returnedByte );
+
+        final int returnedInt = parser.getInteger();
+
+        assertEquals( "Integer should be equal", setInteger, returnedInt );
+
+        final byte[] returnedByteArray = parser.getBytes();
+
+        assertArrayEquals( "arrays should be equal", setByteArray, returnedByteArray );
+    }
+
+
+    @Test
+    public void largerThanUnsignedShorts() {
+        final int maxShortSize = 65535;
+
+        final int lengthOfArray = maxShortSize + 1;
+
+        final byte setByte = 2;
+        final int setInteger = 400;
+
+
+        final byte[] setByteArray = generateByteArray( lengthOfArray );
+
+        FieldBufferBuilder builder = new FieldBufferBuilder( 1 );
+
+        builder.addBytes( setByteArray );
+        builder.addByte( setByte );
+        builder.addInteger( setInteger );
+
+        final FieldBufferSerializer serializer = new FieldBufferSerializer();
+
+        final ByteBuffer serialized = serializer.toByteBuffer( builder.build() );
+
+
+        final FieldBuffer parsed = serializer.fromByteBuffer( serialized );
+
+        FieldBufferParser parser = new FieldBufferParser( parsed );
+
+        final byte[] returnedArray = parser.getBytes();
+
+        assertArrayEquals( setByteArray, returnedArray );
+
+        final byte returnedByte = parser.getByte();
+
+        assertEquals( "Bytes should be equal", setByte, returnedByte );
+
+        final int returnedInt = parser.getInteger();
+
+        assertEquals( "Integer should be equal", setInteger, returnedInt );
+    }
+
+
+    private byte[] generateByteArray( final int length ) {
+        final byte[] data = new byte[length];
+
+        //set it to something other than 0 so we can be sure we're allocating correctly on parse
+        Arrays.fill( data, ( byte ) 1 );
+
+        return data;
+    }
+}