You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2010/02/19 22:45:11 UTC
svn commit: r911995 [2/2] - in /hadoop/avro/trunk/lang/java/src:
java/org/apache/avro/file/ java/org/apache/avro/io/
java/org/apache/avro/ipc/ java/org/apache/avro/tool/
test/java/org/apache/avro/ test/java/org/apache/avro/io/
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java Fri Feb 19 21:45:10 2010
@@ -17,7 +17,6 @@
*/
package org.apache.avro.io;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
@@ -35,20 +34,26 @@
* Avro encoding and decoding.
*/
public class Perf {
- private static final int COUNT = 200000; // needs to be a multiple of 4
- private static final int CYCLES = 150;
+ private static final int COUNT = 100000; // needs to be a multiple of 4
+ private static final int CYCLES = 500;
public static void main(String[] args) throws IOException {
List<Test> tests = new ArrayList<Test>();
+ ReadInt readIntTest = null;
for (String a : args) {
if (a.equals("-i")) {
- tests.add(new ReadInt());
+ readIntTest = new ReadInt();
+ tests.add(readIntTest);
} else if (a.equals("-f")) {
tests.add(new ReadFloat());
} else if (a.equals("-d")) {
tests.add(new ReadDouble());
} else if (a.equals("-l")) {
tests.add(new ReadLong());
+ } else if (a.equals("-ls")) {
+ tests.add(new ReadLongSmall(readIntTest));
+ } else if (a.equals("-b")) {
+ tests.add(new ReadBoolean());
} else if (a.equals("-R")) {
tests.add(new RepeaterTest());
} else if (a.equals("-N")) {
@@ -71,32 +76,47 @@
}
}
if (tests.isEmpty()) {
+ readIntTest = new ReadInt();
tests.addAll(Arrays.asList(new Test[] {
- new ReadInt(), new ReadLong(),
- new ReadFloat(), new ReadDouble(),
+ readIntTest,
+ new ReadLongSmall(readIntTest),
+ new ReadLong(),
+ new ReadFloat(),
+ new ReadDouble(),
+ new ReadBoolean(),
new RepeaterTest(), new NestedRecordTest(),
new ResolverTest(), new MigrationTest(),
new GenericReaderTest(), new GenericReaderWithDefaultTest(),
new GenericReaderWithOutOfOrderTest(),
- new GenericReaderWithPromotionTest(),
+ new GenericReaderWithPromotionTest()
}));
}
- for (Test t : tests) {
+ for (int k = 0; k < tests.size(); k++) {
+ Test t = tests.get(k);
+ // get everything to compile once
+ t.read();
+ }
+ for (int k = 0; k < tests.size(); k++) {
+ Test t = tests.get(k);
// warmup JVM
- for (int i = 0; i < CYCLES; i++) {
+ for (int i = 0; i < t.cycles; i++) {
t.read();
}
// test
long s = 0;
- for (int i = 0; i < CYCLES; i++) {
+ for (int i = 0; i < t.cycles; i++) {
long l = t.read();
// System.out.println("** " + l);
s += l;
}
s /= 1000;
+ double entries = (t.cycles * (double) t.count);
+ double bytes = t.cycles * (double) t.data.length;
System.out.println(t.name + ": " + (s / 1000) + " ms, "
- + ((CYCLES * (double) COUNT) / s) + " million entries/sec");
+ + (entries / s) + " million entries/sec. "
+ + (bytes / s) + " million bytes/sec" );
+ tests.set(k, null);
}
}
@@ -106,6 +126,10 @@
* Name of the test.
*/
public final String name;
+ public final int count;
+ public final int cycles;
+ protected byte[] data;
+ protected static DecoderFactory factory = new DecoderFactory();
/**
* Reads the contents and returns the time taken in nanoseconds.
@@ -114,20 +138,40 @@
*/
abstract long read() throws IOException;
- public Test(String name) {
+ public Test(String name, int cycles, int count) {
this.name = name;
+ this.cycles = cycles;
+ this.count = count;
+ }
+
+ protected void generateRepeaterData(Encoder e) throws IOException {
+ e.writeArrayStart();
+ e.setItemCount(count);
+ Random r = newRandom();
+ for (int i = 0; i < count; i++) {
+ e.writeDouble(r.nextDouble());
+ e.writeDouble(r.nextDouble());
+ e.writeDouble(r.nextDouble());
+ e.writeInt(r.nextInt());
+ e.writeInt(r.nextInt());
+ e.writeInt(r.nextInt());
+ }
+ e.writeArrayEnd();
}
}
private static abstract class DecoderTest extends Test {
public final Schema schema;
- protected byte[] data;
public DecoderTest(String name, String json) throws IOException {
- super(name);
+ this(name, json, 1);
+ }
+ public DecoderTest(String name, String json, int factor) throws IOException {
+ super(name, CYCLES, COUNT/factor);
this.schema = Schema.parse(json);
ByteArrayOutputStream bao = new ByteArrayOutputStream();
Encoder e = new BinaryEncoder(bao);
genData(e);
+ e.flush();
data = bao.toByteArray();
}
@@ -148,7 +192,7 @@
}
protected static Decoder newDecoder(byte[] data) {
- return new BinaryDecoder(new ByteArrayInputStream(data));
+ return factory.createBinaryDecoder(data, null);
}
abstract void genData(Encoder e) throws IOException;
@@ -176,9 +220,9 @@
@Override void genData(Encoder e) throws IOException {
e.writeArrayStart();
- e.setItemCount((COUNT/4) * 4); //next lowest multiple of 4
+ e.setItemCount((count/4) * 4); //next lowest multiple of 4
Random r = newRandom();
- for (int i = 0; i < COUNT/4; i++) {
+ for (int i = 0; i < count/4; i++) {
e.writeInt(r.nextInt(50)); // fits in 1 byte
e.writeInt(r.nextInt(5000)); // fits in 2 bytes
e.writeInt(r.nextInt(500000)); // fits in 3 bytes
@@ -193,6 +237,21 @@
}
}
+ // This is the same data as ReadInt, but using readLong.
+ private static class ReadLongSmall extends DecoderTest {
+ public ReadLongSmall(ReadInt dataFrom) throws IOException {
+ super("ReadLongSmall", "{ \"type\": \"array\", \"items\": \"long\"} ");
+ data = dataFrom.data;
+ }
+ @Override void genData(Encoder e) throws IOException {
+ }
+ @Override
+ void readInternal(Decoder d) throws IOException {
+ d.readLong();
+ }
+ }
+
+ // this tests reading Longs that are sometimes very large
private static class ReadLong extends DecoderTest {
public ReadLong() throws IOException {
super("ReadLong", "{ \"type\": \"array\", \"items\": \"long\"} ");
@@ -201,9 +260,9 @@
@Override
void genData(Encoder e) throws IOException {
e.writeArrayStart();
- e.setItemCount((COUNT / 4) *4);
+ e.setItemCount((count / 4) *4);
Random r = newRandom();
- for (int i = 0; i < COUNT /4; i++) {
+ for (int i = 0; i < count /4; i++) {
e.writeLong(r.nextLong() % 0x7FL); // half fit in 1, half in 2
e.writeLong(r.nextLong() % 0x1FFFFFL); // half fit in <=3, half in 4
e.writeLong(r.nextLong() % 0x3FFFFFFFFL); // half in <=5, half in 6
@@ -226,9 +285,9 @@
@Override
void genData(Encoder e) throws IOException {
e.writeArrayStart();
- e.setItemCount(COUNT);
+ e.setItemCount(count);
Random r = newRandom();
- for (int i = 0; i < COUNT; i++) {
+ for (int i = 0; i < count; i++) {
e.writeFloat(r.nextFloat());
}
e.writeArrayEnd();
@@ -240,6 +299,25 @@
}
}
+ private static class ReadBoolean extends DecoderTest {
+ public ReadBoolean() throws IOException {
+ super("ReadBoolean", "{ \"type\": \"array\", \"items\": \"boolean\"} ");
+ }
+ @Override void genData(Encoder e) throws IOException {
+ e.writeArrayStart();
+ e.setItemCount(count);
+ Random r = newRandom();
+ for (int i = 0; i < count; i++) {
+ e.writeBoolean(r.nextBoolean());
+ }
+ e.writeArrayEnd();
+ }
+ @Override
+ void readInternal(Decoder d) throws IOException {
+ d.readBoolean();
+ }
+ }
+
private static class ReadDouble extends DecoderTest {
public ReadDouble() throws IOException {
super("ReadDouble", "{ \"type\": \"array\", \"items\": \"double\"} ");
@@ -248,9 +326,9 @@
@Override
void genData(Encoder e) throws IOException {
e.writeArrayStart();
- e.setItemCount(COUNT);
+ e.setItemCount(count);
Random r = newRandom();
- for (int i = 0; i < COUNT; i++) {
+ for (int i = 0; i < count; i++) {
e.writeDouble(r.nextFloat());
}
e.writeArrayEnd();
@@ -272,27 +350,13 @@
+ "{ \"name\": \"f6\", \"type\": \"int\" }\n"
+ "] } }";
- private static void generateRepeaterData(Encoder e) throws IOException {
- e.writeArrayStart();
- e.setItemCount(COUNT);
- Random r = newRandom();
- for (int i = 0; i < COUNT; i++) {
- e.writeDouble(r.nextDouble());
- e.writeDouble(r.nextDouble());
- e.writeDouble(r.nextDouble());
- e.writeInt(r.nextInt());
- e.writeInt(r.nextInt());
- e.writeInt(r.nextInt());
- }
- e.writeArrayEnd();
- }
private static class RepeaterTest extends DecoderTest {
public RepeaterTest() throws IOException {
this("RepeaterTest");
}
public RepeaterTest(String name) throws IOException {
- super(name, REPEATER_SCHEMA);
+ super(name, REPEATER_SCHEMA, 6);
}
@Override
@@ -401,14 +465,13 @@
private static class GenericReaderTest extends Test {
public final Schema writerSchema;
- protected byte[] data;
public GenericReaderTest() throws IOException {
this("GenericReaderTest");
}
public GenericReaderTest(String name) throws IOException {
- super(name);
+ super(name, CYCLES, COUNT/12);
this.writerSchema = Schema.parse(REPEATER_SCHEMA);
ByteArrayOutputStream bao = new ByteArrayOutputStream();
Encoder e = new BinaryEncoder(bao);
@@ -421,10 +484,11 @@
GenericDatumReader<Object> r = getReader();
long t = System.nanoTime();
Decoder d =
- new BinaryDecoder(new ByteArrayInputStream(data));
+ DecoderFactory.defaultFactory().createBinaryDecoder(data, null);
+ Object reuse = null;
for (; ;) {
try {
- r.read(null, d);
+ reuse = r.read(reuse, d);
} catch (EOFException e) {
break;
}
@@ -490,11 +554,14 @@
}
private static void usage() {
- System.out.println("Usage: Perf { -i | -l | -f | -d }");
+ System.out.println("Usage: Perf { -i | -ls | -l | -f | -d | -b | -R | -N " +
+ "| -S | -M | -G | -Gd | -Go | Gp }");
System.out.println(" -i readInt()");
+ System.out.println(" -ls readLongSmall()");
System.out.println(" -l readLong()");
System.out.println(" -f readFloat()");
System.out.println(" -d readDouble()");
+ System.out.println(" -b readBoolean()");
System.out.println(" -R repeater in validating decoder");
System.out.println(" -N nested record in validating decoder");
System.out.println(" -S resolving decoder");
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBinaryDecoder.java Fri Feb 19 21:45:10 2010
@@ -18,61 +18,385 @@
package org.apache.avro.io;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import org.apache.avro.RandomData;
+import org.apache.avro.Schema;
+import org.apache.avro.generic.GenericDatumReader;
+import org.apache.avro.generic.GenericDatumWriter;
import org.apache.avro.util.Utf8;
+import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
-
+@RunWith(Parameterized.class)
public class TestBinaryDecoder {
+ // prime number buffer size so that looping tests hit the buffer edge
+ // at different points in the loop.
+ DecoderFactory factory;
+ public TestBinaryDecoder(boolean useDirect) {
+ factory = new DecoderFactory().configureDecoderBufferSize(521);
+ factory.configureDirectDecoder(useDirect);
+ }
+
+ @Parameters
+ public static Collection<Object[]> data() {
+ return Arrays.asList(new Object[][] {
+ { true },
+ { false },
+ });
+ }
+
+ private Decoder newDecoderWithNoData() throws IOException {
+ return newDecoder(new byte[0]);
+ }
+
+ private Decoder newDecoder(byte[] bytes, int start, int len)
+ throws IOException {
+ return factory.createBinaryDecoder(bytes, start, len, null);
+
+ }
+
+ private Decoder newDecoder(InputStream in) {
+ return factory.createBinaryDecoder(in, null);
+ }
+
+ private Decoder newDecoder(byte[] bytes) throws IOException {
+ return factory.createBinaryDecoder(bytes, null);
+ }
+
/** Verify EOFException throw at EOF */
@Test(expected=EOFException.class)
public void testEOFBoolean() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readBoolean();
+ newDecoderWithNoData().readBoolean();
}
@Test(expected=EOFException.class)
public void testEOFInt() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readInt();
+ newDecoderWithNoData().readInt();
}
@Test(expected=EOFException.class)
public void testEOFLong() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readLong();
+ newDecoderWithNoData().readLong();
}
@Test(expected=EOFException.class)
public void testEOFFloat() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readFloat();
+ newDecoderWithNoData().readFloat();
}
@Test(expected=EOFException.class)
public void testEOFDouble() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readDouble();
+ newDecoderWithNoData().readDouble();
}
@Test(expected=EOFException.class)
public void testEOFBytes() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readBytes(null);
+ newDecoderWithNoData().readBytes(null);
}
@Test(expected=EOFException.class)
public void testEOFString() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).
- readString(new Utf8("a"));
+ newDecoderWithNoData().readString(new Utf8("a"));
}
@Test(expected=EOFException.class)
public void testEOFFixed() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).
- readFixed(new byte[1]);
+ newDecoderWithNoData().readFixed(new byte[1]);
}
@Test(expected=EOFException.class)
public void testEOFEnum() throws IOException {
- new BinaryDecoder(new ByteArrayInputStream(new byte[0])).readEnum();
+ newDecoderWithNoData().readEnum();
}
+ private static byte[] data = null;
+ private static int seed = -1;
+ private static Schema schema = null;
+ private static int count = 200;
+ private static ArrayList<Object> records = new ArrayList<Object>(count);
+ @BeforeClass
+ public static void generateData() throws IOException {
+ seed = (int)System.currentTimeMillis();
+ // note some tests (testSkipping) rely on this explicitly
+ String jsonSchema =
+ "{\"type\": \"record\", \"name\": \"Test\", \"fields\": ["
+ +"{\"name\":\"intField\", \"type\":\"int\"},"
+ +"{\"name\":\"bytesField\", \"type\":\"bytes\"},"
+ +"{\"name\":\"booleanField\", \"type\":\"boolean\"},"
+ +"{\"name\":\"stringField\", \"type\":\"string\"},"
+ +"{\"name\":\"floatField\", \"type\":\"float\"},"
+ +"{\"name\":\"doubleField\", \"type\":\"double\"},"
+ +"{\"name\":\"arrayField\", \"type\": " +
+ "{\"type\":\"array\", \"items\":\"boolean\"}},"
+ +"{\"name\":\"longField\", \"type\":\"long\"}]}";
+ schema = Schema.parse(jsonSchema);
+ GenericDatumWriter<Object> writer = new GenericDatumWriter<Object>();
+ writer.setSchema(schema);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(8192);
+ BinaryEncoder encoder = new BinaryEncoder(baos);
+
+ for (Object datum : new RandomData(schema, count, seed)) {
+ writer.write(datum, encoder);
+ records.add(datum);
+ }
+ data = baos.toByteArray();
+ }
+
+ @Test
+ public void testDecodeFromSources() throws IOException {
+ GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
+ reader.setSchema(schema);
+
+ ByteArrayInputStream is = new ByteArrayInputStream(data);
+ ByteArrayInputStream is2 = new ByteArrayInputStream(data);
+ ByteArrayInputStream is3 = new ByteArrayInputStream(data);
+
+ Decoder fromInputStream = newDecoder(is);
+ Decoder fromArray = newDecoder(data);
+
+ byte[] data2 = new byte[data.length + 30];
+ Arrays.fill(data2, (byte)0xff);
+ System.arraycopy(data, 0, data2, 15, data.length);
+
+ Decoder fromOffsetArray = newDecoder(data2, 15, data.length);
+
+ BinaryDecoder initOnInputStream = factory.createBinaryDecoder(
+ new byte[50], 0, 30, null);
+ initOnInputStream = factory.createBinaryDecoder(is2, initOnInputStream);
+ BinaryDecoder initOnArray = factory.createBinaryDecoder(is3, null);
+ initOnArray = factory.createBinaryDecoder(
+ data, 0, data.length, initOnArray);
+
+ for (Object datum : records) {
+ Assert.assertEquals(
+ "InputStream based BinaryDecoder result does not match",
+ datum, reader.read(null, fromInputStream));
+ Assert.assertEquals(
+ "Array based BinaryDecoder result does not match",
+ datum, reader.read(null, fromArray));
+ Assert.assertEquals(
+ "offset Array based BinaryDecoder result does not match",
+ datum, reader.read(null, fromOffsetArray));
+ Assert.assertEquals(
+ "InputStream initialized BinaryDecoder result does not match",
+ datum, reader.read(null, initOnInputStream));
+ Assert.assertEquals(
+ "Array initialized BinaryDecoder result does not match",
+ datum, reader.read(null, initOnArray));
+ }
+ }
+
+ @Test
+ public void testInputStreamProxy() throws IOException {
+ Decoder d = newDecoder(data);
+ if (d instanceof BinaryDecoder) {
+ BinaryDecoder bd = (BinaryDecoder) d;
+ InputStream test = bd.inputStream();
+ InputStream check = new ByteArrayInputStream(data);
+ validateInputStreamReads(test, check);
+ bd = factory.createBinaryDecoder(data, bd);
+ test = bd.inputStream();
+ check = new ByteArrayInputStream(data);
+ validateInputStreamSkips(test, check);
+ // with input stream sources
+ bd = factory.createBinaryDecoder(new ByteArrayInputStream(data), bd);
+ test = bd.inputStream();
+ check = new ByteArrayInputStream(data);
+ validateInputStreamReads(test, check);
+ bd = factory.createBinaryDecoder(new ByteArrayInputStream(data), bd);
+ test = bd.inputStream();
+ check = new ByteArrayInputStream(data);
+ validateInputStreamSkips(test, check);
+ }
+ }
+
+ @Test
+ public void testInputStreamProxyDetached() throws IOException {
+ Decoder d = newDecoder(data);
+ if (d instanceof BinaryDecoder) {
+ BinaryDecoder bd = (BinaryDecoder) d;
+ InputStream test = bd.inputStream();
+ InputStream check = new ByteArrayInputStream(data);
+ // detach input stream and decoder from old source
+ factory.createBinaryDecoder(new byte[56], null);
+ InputStream bad = bd.inputStream();
+ InputStream check2 = new ByteArrayInputStream(data);
+ validateInputStreamReads(test, check);
+ Assert.assertFalse(bad.read() == check2.read());
+ }
+ }
+
+ @Test
+ public void testInputStreamPartiallyUsed() throws IOException {
+ BinaryDecoder bd = factory.createBinaryDecoder(
+ new ByteArrayInputStream(data), null);
+ InputStream test = bd.inputStream();
+ InputStream check = new ByteArrayInputStream(data);
+ // triggers buffer fill if unused and tests isEnd()
+ try {
+ Assert.assertFalse(bd.isEnd());
+ } catch (UnsupportedOperationException e) {
+ // this is ok if its a DirectBinaryDecoder.
+ if (bd.getClass() != DirectBinaryDecoder.class) {
+ throw e;
+ }
+ }
+ bd.readFloat(); // use data, and otherwise trigger buffer fill
+ check.skip(4); // skip the same # of bytes here
+ validateInputStreamReads(test, check);
+ }
+
+ private void validateInputStreamReads(InputStream test, InputStream check)
+ throws IOException {
+ byte[] bt = new byte[7];
+ byte[] bc = new byte[7];
+ while (true) {
+ int t = test.read();
+ int c = check.read();
+ Assert.assertEquals(c, t);
+ if (-1 == t) break;
+ t = test.read(bt);
+ c = check.read(bc);
+ Assert.assertEquals(c, t);
+ Assert.assertArrayEquals(bt, bc);
+ if (-1 == t) break;
+ t = test.read(bt, 1, 4);
+ c = check.read(bc, 1, 4);
+ Assert.assertEquals(c, t);
+ Assert.assertArrayEquals(bt, bc);
+ if (-1 == t) break;
+ }
+ Assert.assertEquals(0, test.skip(5));
+ Assert.assertEquals(0, test.available());
+ Assert.assertFalse(test.getClass() != ByteArrayInputStream.class && test.markSupported());
+ test.close();
+ }
+
+ private void validateInputStreamSkips(InputStream test, InputStream check) throws IOException {
+ while(true) {
+ long t2 = test.skip(19);
+ long c2 = check.skip(19);
+ Assert.assertEquals(c2, t2);
+ if (0 == t2) break;
+ }
+ Assert.assertEquals(-1, test.read());
+ }
+
+ @Test
+ public void testBadIntEncoding() throws IOException {
+ byte[] badint = new byte[5];
+ Arrays.fill(badint, (byte)0xff);
+ Decoder bd = factory.createBinaryDecoder(badint, null);
+ String message = "";
+ try {
+ bd.readInt();
+ } catch (IOException ioe) {
+ message = ioe.getMessage();
+ }
+ Assert.assertEquals("Invalid int encoding", message);
+ }
+
+ @Test
+ public void testBadLongEncoding() throws IOException {
+ byte[] badint = new byte[10];
+ Arrays.fill(badint, (byte)0xff);
+ Decoder bd = factory.createBinaryDecoder(badint, null);
+ String message = "";
+ try {
+ bd.readLong();
+ } catch (IOException ioe) {
+ message = ioe.getMessage();
+ }
+ Assert.assertEquals("Invalid long encoding", message);
+ }
+
+ @Test(expected=EOFException.class)
+ public void testIntTooShort() throws IOException {
+ byte[] badint = new byte[4];
+ Arrays.fill(badint, (byte)0xff);
+ newDecoder(badint).readInt();
+ }
+
+ @Test(expected=EOFException.class)
+ public void testLongTooShort() throws IOException {
+ byte[] badint = new byte[9];
+ Arrays.fill(badint, (byte)0xff);
+ newDecoder(badint).readLong();
+ }
+
+ @Test(expected=EOFException.class)
+ public void testFloatTooShort() throws IOException {
+ byte[] badint = new byte[3];
+ Arrays.fill(badint, (byte)0xff);
+ newDecoder(badint).readInt();
+ }
+
+ @Test(expected=EOFException.class)
+ public void testDoubleTooShort() throws IOException {
+ byte[] badint = new byte[7];
+ Arrays.fill(badint, (byte)0xff);
+ newDecoder(badint).readLong();
+ }
+
+ @Test
+ public void testSkipping() throws IOException {
+ Decoder d = newDecoder(data);
+ skipGenerated(d);
+ if (d instanceof BinaryDecoder) {
+ BinaryDecoder bd = (BinaryDecoder) d;
+ try {
+ Assert.assertTrue(bd.isEnd());
+ } catch (UnsupportedOperationException e) {
+ // this is ok if its a DirectBinaryDecoder.
+ if (bd.getClass() != DirectBinaryDecoder.class) {
+ throw e;
+ }
+ }
+ bd = factory.createBinaryDecoder(new ByteArrayInputStream(data), bd);
+ skipGenerated(bd);
+ try {
+ Assert.assertTrue(bd.isEnd());
+ } catch (UnsupportedOperationException e) {
+ // this is ok if its a DirectBinaryDecoder.
+ if (bd.getClass() != DirectBinaryDecoder.class) {
+ throw e;
+ }
+ }
+ }
+ }
+
+ private void skipGenerated(Decoder bd) throws IOException {
+ for (int i = 0; i < records.size(); i++) {
+ bd.readInt();
+ bd.skipBytes();
+ bd.skipFixed(1);
+ bd.skipString();
+ bd.skipFixed(4);
+ bd.skipFixed(8);
+ long leftover = bd.skipArray();
+ // booleans are one byte, array trailer is one byte
+ bd.skipFixed((int)leftover + 1);
+ bd.skipFixed(0);
+ bd.readLong();
+ }
+ EOFException eof = null;
+ try {
+ bd.skipFixed(4);
+ } catch (EOFException e) {
+ eof = e;
+ }
+ Assert.assertTrue(null != eof);
+ }
}
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO.java Fri Feb 19 21:45:10 2010
@@ -68,7 +68,7 @@
byte[] bb = os.toByteArray();
// dump(bb);
- this.input = new BinaryDecoder(new ByteArrayInputStream(bb));
+ this.input = DecoderFactory.defaultFactory().createBinaryDecoder(bb, null);
this.parser = f.createJsonParser(new ByteArrayInputStream(in));
}
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestBlockingIO2.java Fri Feb 19 21:45:10 2010
@@ -17,10 +17,8 @@
*/
package org.apache.avro.io;
-import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.util.Collection;
import java.util.Arrays;
@@ -53,8 +51,7 @@
byte[] bb = os.toByteArray();
- InputStream in = new ByteArrayInputStream(bb);
- decoder = new BinaryDecoder(in);
+ decoder = DecoderFactory.defaultFactory().createBinaryDecoder(bb, null);
this.calls = calls;
}
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestResolvingIO.java Fri Feb 19 21:45:10 2010
@@ -97,14 +97,14 @@
throws IOException {
// TestValidatingIO.dump(bytes);
// System.out.println(new String(bytes, "UTF-8"));
- InputStream in = new ByteArrayInputStream(bytes);
Decoder bvi = null;
switch (encoding) {
case BINARY:
case BLOCKING_BINARY:
- bvi = new BinaryDecoder(in);
+ bvi = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
break;
case JSON:
+ InputStream in = new ByteArrayInputStream(bytes);
bvi = new JsonDecoder(wsc, in);
break;
}
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java?rev=911995&r1=911994&r2=911995&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/TestValidatingIO.java Fri Feb 19 21:45:10 2010
@@ -298,14 +298,14 @@
throws IOException {
// dump(bytes);
// System.out.println(new String(bytes, "UTF-8"));
- InputStream in = new ByteArrayInputStream(bytes);
Decoder bvi = null;
switch (encoding) {
case BINARY:
case BLOCKING_BINARY:
- bvi = new BinaryDecoder(in);
+ bvi = DecoderFactory.defaultFactory().createBinaryDecoder(bytes, null);
break;
case JSON:
+ InputStream in = new ByteArrayInputStream(bytes);
bvi = new JsonDecoder(sc, in);
}
Decoder vi = new ValidatingDecoder(sc, bvi);