You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2013/01/18 20:34:34 UTC
svn commit: r1435314 - in /avro/trunk: ./
lang/java/tools/src/main/java/org/apache/avro/tool/
lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/
lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/
lang/java/trevni/core/src/main/java/or...
Author: cutting
Date: Fri Jan 18 19:34:34 2013
New Revision: 1435314
URL: http://svn.apache.org/viewvc?rev=1435314&view=rev
Log:
AVRO-1229. Add support for booleans to Trevni.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java
avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java
avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
avro/trunk/lang/java/trevni/doc/apt/spec.apt
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jan 18 19:34:34 2013
@@ -4,6 +4,8 @@ Trunk (not yet released)
NEW FEATURES
+ AVRO-1229. Add support for booleans to Trevni. (cutting)
+
IMPROVEMENTS
AVRO-1211. Add MR guide to documentation. (Skye Wanderman-Milne via
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TrevniToJsonTool.java Fri Jan 18 19:34:34 2013
@@ -143,6 +143,8 @@ public class TrevniToJsonTool implements
switch (column.getType()) {
case NULL:
generator.writeNull(); break;
+ case BOOLEAN:
+ generator.writeBoolean((Boolean)value); break;
case INT:
generator.writeNumber((Integer)value); break;
case LONG:
Modified: avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java (original)
+++ avro/trunk/lang/java/trevni/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java Fri Jan 18 19:34:34 2013
@@ -137,7 +137,7 @@ class AvroColumnator {
static boolean isSimple(Schema s) {
switch (s.getType()) {
- case NULL:
+ case NULL: case BOOLEAN:
case INT: case LONG:
case FLOAT: case DOUBLE:
case BYTES: case STRING:
@@ -151,6 +151,7 @@ class AvroColumnator {
private ValueType simpleValueType(Schema s) {
switch (s.getType()) {
case NULL: return ValueType.NULL;
+ case BOOLEAN:return ValueType.BOOLEAN;
case INT: return ValueType.INT;
case LONG: return ValueType.LONG;
case FLOAT: return ValueType.FLOAT;
Modified: avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java (original)
+++ avro/trunk/lang/java/trevni/avro/src/test/java/org/apache/trevni/avro/TestShredder.java Fri Jan 18 19:34:34 2013
@@ -38,6 +38,8 @@ public class TestShredder {
@Test public void testPrimitives() throws Exception {
check(Schema.create(Schema.Type.NULL),
new ColumnMetaData("null", ValueType.NULL));
+ check(Schema.create(Schema.Type.BOOLEAN),
+ new ColumnMetaData("boolean", ValueType.BOOLEAN));
check(Schema.create(Schema.Type.INT),
new ColumnMetaData("int", ValueType.INT));
Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ArrayColumnOutputBuffer.java Fri Jan 18 19:34:34 2013
@@ -33,7 +33,7 @@ class ArrayColumnOutputBuffer extends Co
@Override public void writeLength(int length) throws IOException {
assert this.length == 0;
this.length = length;
- getBuffer().writeInt(length);
+ getBuffer().writeLength(length);
}
@Override public void writeValue(Object value) throws IOException {
Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ColumnValues.java Fri Jan 18 19:34:34 2013
@@ -136,7 +136,7 @@ public class ColumnValues<T extends Comp
throw new TrevniRuntimeException
("Column is not array: " +column.metaData.getName());
assert arrayLength == 0;
- return arrayLength = values.readInt();
+ return arrayLength = values.readLength();
}
/** Expert: Returns the next value in a column. */
Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/InputBuffer.java Fri Jan 18 19:34:34 2013
@@ -33,6 +33,8 @@ class InputBuffer {
private int pos; // position within buffer
private int limit; // end of valid buffer data
+ private int bitCount; // position in booleans
+
public InputBuffer(Input in) throws IOException { this(in, 0); }
public InputBuffer(Input in, long position) throws IOException {
@@ -68,6 +70,8 @@ class InputBuffer {
switch (type) {
case NULL:
return (T)null;
+ case BOOLEAN:
+ return (T)Boolean.valueOf(readBoolean());
case INT:
return (T)Integer.valueOf(readInt());
case LONG:
@@ -93,6 +97,8 @@ class InputBuffer {
switch (type) {
case NULL:
break;
+ case BOOLEAN:
+ readBoolean(); break;
case INT:
readInt(); break;
case LONG:
@@ -111,6 +117,22 @@ class InputBuffer {
}
}
+ public boolean readBoolean() throws IOException {
+ if (bitCount == 0)
+ read();
+ int bits = buf[pos-1] & 0xff;
+ int bit = (bits >> bitCount) & 1;
+ bitCount++;
+ if (bitCount == 8)
+ bitCount = 0;
+ return bit == 0 ? false : true;
+ }
+
+ public int readLength() throws IOException {
+ bitCount = 0;
+ return readInt();
+ }
+
public int readInt() throws IOException {
if ((limit - pos) < 5) { // maybe not in buffer
int b = read();
Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/OutputBuffer.java Fri Jan 18 19:34:34 2013
@@ -27,6 +27,8 @@ import java.util.Arrays;
class OutputBuffer extends ByteArrayOutputStream {
static final int BLOCK_SIZE = 64 * 1024;
+ private int bitCount; // position in booleans
+
public OutputBuffer() { super(BLOCK_SIZE + BLOCK_SIZE >> 2); }
public boolean isFull() { return size() >= BLOCK_SIZE; }
@@ -38,6 +40,8 @@ class OutputBuffer extends ByteArrayOutp
switch (type) {
case NULL:
break;
+ case BOOLEAN:
+ writeBoolean((Boolean)value); break;
case INT:
writeInt((Integer)value); break;
case LONG:
@@ -63,6 +67,23 @@ class OutputBuffer extends ByteArrayOutp
}
}
+ public void writeBoolean(boolean value) {
+ if (bitCount == 0) { // first bool in byte
+ ensure(1);
+ count++;
+ }
+ if (value)
+ buf[count-1] |= (byte)(1 << bitCount);
+ bitCount++;
+ if (bitCount == 8)
+ bitCount = 0;
+ }
+
+ public void writeLength(int length) throws IOException {
+ bitCount = 0;
+ writeInt(length);
+ }
+
private static final Charset UTF8 = Charset.forName("UTF-8");
public void writeString(String string) throws IOException {
Modified: avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java (original)
+++ avro/trunk/lang/java/trevni/core/src/main/java/org/apache/trevni/ValueType.java Fri Jan 18 19:34:34 2013
@@ -19,7 +19,7 @@ package org.apache.trevni;
/** The datatypes that may be stored in a column. */
public enum ValueType {
- NULL, INT, LONG, FIXED32, FIXED64, FLOAT, DOUBLE, STRING, BYTES;
+ NULL, BOOLEAN, INT, LONG, FIXED32, FIXED64, FLOAT, DOUBLE, STRING, BYTES;
private String name;
private ValueType() { this.name = this.name().toLowerCase(); }
Modified: avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java (original)
+++ avro/trunk/lang/java/trevni/core/src/test/java/org/apache/trevni/TestIOBuffers.java Fri Jan 18 19:34:34 2013
@@ -26,7 +26,7 @@ import org.junit.Test;
public class TestIOBuffers {
- private static final int COUNT = 1000;
+ private static final int COUNT = 1001;
@Test public void testEmpty() throws Exception {
OutputBuffer out = new OutputBuffer();
@@ -47,6 +47,19 @@ public class TestIOBuffers {
Assert.assertEquals(0, in.readInt());
}
+ @Test public void testBoolean() throws Exception {
+ Random random = TestUtil.createRandom();
+ OutputBuffer out = new OutputBuffer();
+ for (int i = 0; i < COUNT; i++)
+ out.writeValue(random.nextBoolean(), ValueType.BOOLEAN);
+
+ InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
+ random = TestUtil.createRandom();
+ for (int i = 0; i < COUNT; i++)
+ Assert.assertEquals(random.nextBoolean(),
+ in.readValue(ValueType.BOOLEAN));
+ }
+
@Test public void testInt() throws Exception {
Random random = TestUtil.createRandom();
OutputBuffer out = new OutputBuffer();
@@ -150,6 +163,16 @@ public class TestIOBuffers {
in.skipValue(ValueType.NULL);
Assert.assertEquals(sentinel, in.readLong());
}
+ @Test public void testSkipBoolean() throws Exception {
+ long sentinel = Long.MAX_VALUE;
+ OutputBuffer out = new OutputBuffer();
+ out.writeValue(false, ValueType.BOOLEAN);
+ out.writeLong(sentinel);
+
+ InputBuffer in = new InputBuffer(new InputBytes(out.toByteArray()));
+ in.skipValue(ValueType.BOOLEAN);
+ Assert.assertEquals(sentinel, in.readLong());
+ }
@Test public void testSkipInt() throws Exception {
long sentinel = Long.MAX_VALUE;
OutputBuffer out = new OutputBuffer();
Modified: avro/trunk/lang/java/trevni/doc/apt/spec.apt
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/trevni/doc/apt/spec.apt?rev=1435314&r1=1435313&r2=1435314&view=diff
==============================================================================
--- avro/trunk/lang/java/trevni/doc/apt/spec.apt (original)
+++ avro/trunk/lang/java/trevni/doc/apt/spec.apt Fri Jan 18 19:34:34 2013
@@ -190,6 +190,8 @@ whether subsequent bytes are present. F
* <<null>>, requires zero bytes. Sometimes used in array columns.
+ * <<boolean>>, one bit, packed into bytes, little-endian;
+
* <<int>>, like <long>, but restricted to 32-bit signed values
* <<long>> 64-bit signed values, represented as above