You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2009/06/05 20:31:45 UTC
svn commit: r782092 [2/2] - in /hadoop/avro/trunk: ./
src/java/org/apache/avro/file/ src/java/org/apache/avro/generic/
src/java/org/apache/avro/io/ src/java/org/apache/avro/ipc/
src/test/java/org/apache/avro/ src/test/java/org/apache/avro/io/
Added: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java?rev=782092&view=auto
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java (added)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestBlockingIO.java Fri Jun 5 18:31:45 2009
@@ -0,0 +1,450 @@
+package org.apache.avro.io;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.Stack;
+
+import org.codehaus.jackson.JsonFactory;
+import org.codehaus.jackson.JsonParseException;
+import org.codehaus.jackson.JsonParser;
+import org.testng.Assert;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class TestBlockingIO {
+ private static final String UTF_8 = "UTF-8";
+
+ private static class Tests {
+ private final JsonParser parser;
+ private final ValueReader input;
+ private final int depth;
+ public Tests(int bufferSize, int depth, String input)
+ throws JsonParseException, IOException {
+
+ this.depth = depth;
+ byte[] in = input.getBytes("UTF-8");
+ JsonFactory f = new JsonFactory();
+ JsonParser p = f.createJsonParser(
+ new ByteArrayInputStream(input.getBytes("UTF-8")));
+
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ ValueWriter cos = new BlockingValueWriter(os, bufferSize);
+ serialize(cos, p, os);
+ cos.flush();
+
+ byte[] bb = os.toByteArray();
+ // dump(bb);
+ this.input = new ValueReader(new ByteArrayInputStream(bb));
+ this.parser = f.createJsonParser(new ByteArrayInputStream(in));
+ }
+
+ public void scan()
+ throws JsonParseException, UnsupportedEncodingException, IOException {
+ Stack<S> countStack = new Stack<S>();
+ long count = 0;
+ while (parser.nextToken() != null) {
+ switch (parser.getCurrentToken()) {
+ case END_ARRAY:
+ Assert.assertEquals(0, count);
+ Assert.assertTrue(countStack.peek().isArray);
+ count = countStack.pop().count;
+ break;
+ case END_OBJECT:
+ Assert.assertEquals(0, count);
+ Assert.assertFalse(countStack.peek().isArray);
+ count = countStack.pop().count;
+ break;
+ case START_ARRAY:
+ countStack.push(new S(count, true));
+ count = input.readArrayStart();
+ continue;
+ case VALUE_STRING:
+ {
+ String s = parser.getText();
+ int n = s.getBytes(UTF_8).length;
+ checkString(s, input, n);
+ break;
+ }
+ case FIELD_NAME:
+ {
+ String s = parser.getCurrentName();
+ int n = s.getBytes(UTF_8).length;
+ checkString(s, input, n);
+ continue;
+ }
+ case START_OBJECT:
+ countStack.push(new S(count, false));
+ count = input.readMapStart();
+ if (count < 0) {
+ count = -count;
+ input.readLong(); // byte count
+ }
+ continue;
+ default:
+ throw new RuntimeException("Unsupported: " + parser.getCurrentToken());
+ }
+ count--;
+ if (count == 0) {
+ count = countStack.peek().isArray ? input.arrayNext() :
+ input.mapNext();
+ }
+ }
+ }
+
+ public void skip(int skipLevel) throws
+ JsonParseException, UnsupportedEncodingException, IOException {
+ Stack<S> countStack = new Stack<S>();
+ long count = 0;
+ while (parser.nextToken() != null) {
+ switch (parser.getCurrentToken()) {
+ case END_ARRAY:
+ // assertEquals(0, count);
+ Assert.assertTrue(countStack.peek().isArray);
+ count = countStack.pop().count;
+ break;
+ case END_OBJECT:
+ // assertEquals(0, count);
+ Assert.assertFalse(countStack.peek().isArray);
+ count = countStack.pop().count;
+ break;
+ case START_ARRAY:
+ if (countStack.size() == skipLevel) {
+ skipArray(parser, input, depth - skipLevel);
+ break;
+ } else {
+ countStack.push(new S(count, true));
+ count = input.readArrayStart();
+ continue;
+ }
+ case VALUE_STRING:
+ {
+ if (countStack.size() == skipLevel) {
+ input.skipBytes();
+ } else {
+ String s = parser.getText();
+ int n = s.getBytes(UTF_8).length;
+ checkString(s, input, n);
+ }
+ break;
+ }
+ case FIELD_NAME:
+ {
+ String s = parser.getCurrentName();
+ int n = s.getBytes(UTF_8).length;
+ checkString(s, input, n);
+ continue;
+ }
+ case START_OBJECT:
+ if (countStack.size() == skipLevel) {
+ skipMap(parser, input, depth - skipLevel);
+ break;
+ } else {
+ countStack.push(new S(count, false));
+ count = input.readMapStart();
+ if (count < 0) {
+ count = -count;
+ input.readLong(); // byte count
+ }
+ continue;
+ }
+ default:
+ throw new RuntimeException("Unsupported: " + parser.getCurrentToken());
+ }
+ count--;
+ if (count == 0) {
+ count = countStack.peek().isArray ? input.arrayNext() :
+ input.mapNext();
+ }
+ }
+ }
+ }
+
+ protected static void dump(byte[] bb) {
+ int col = 0;
+ for (byte b : bb) {
+ if (col % 16 == 0) {
+ System.out.println();
+ }
+ col++;
+ System.out.print(Integer.toHexString(b & 0xff) + " ");
+ }
+ System.out.println();
+ }
+
+ private static class S {
+ public final long count;
+ public final boolean isArray;
+
+ public S(long count, boolean isArray) {
+ this.count = count;
+ this.isArray = isArray;
+ }
+ }
+
+ @Test(dataProvider="data")
+ public void testScan(int bufferSize, int depth, String input)
+ throws JsonParseException, IOException {
+ Tests t = new Tests(bufferSize, depth, input);
+ t.scan();
+ }
+
+ @Test(dataProvider="data")
+ public void testSkip_1(int bufferSize, int depth, String input)
+ throws JsonParseException, IOException {
+ testSkip(bufferSize, depth, input, 0);
+ }
+
+ @Test(dataProvider="data")
+ public void testSkip_2(int bufferSize, int depth, String input)
+ throws JsonParseException, IOException {
+ testSkip(bufferSize, depth, input, 1);
+ }
+
+ @Test(dataProvider="data")
+ public void testSkip_3(int bufferSize, int depth, String input)
+ throws JsonParseException, IOException {
+ testSkip(bufferSize, depth, input, 2);
+ }
+
+ private void testSkip(int bufferSize, int depth, String input,
+ int skipLevel)
+ throws JsonParseException, IOException {
+ Tests t = new Tests(bufferSize, depth, input);
+ t.skip(skipLevel);
+ }
+
+ private static void skipMap(JsonParser parser, ValueReader input, int depth)
+ throws IOException, JsonParseException {
+ for (long l = input.skipMap(); l != 0; l = input.skipMap()) {
+ for (long i = 0; i < l; i++) {
+ if (depth == 0) {
+ input.skipBytes();
+ } else {
+ skipArray(parser, input, depth - 1);
+ }
+ }
+ }
+ parser.skipChildren();
+ }
+
+ private static void skipArray(JsonParser parser, ValueReader input, int depth)
+ throws IOException, JsonParseException {
+ for (long l = input.skipArray(); l != 0; l = input.skipArray()) {
+ for (long i = 0; i < l; i++) {
+ if (depth == 1) {
+ input.skipBytes();
+ } else {
+ skipArray(parser, input, depth - 1);
+ }
+ }
+ }
+ parser.skipChildren();
+ }
+
+ private static void checkString(String s, ValueReader input, int n)
+ throws IOException, UnsupportedEncodingException {
+ ByteBuffer buf = input.readBytes(null);
+ Assert.assertEquals(n, buf.remaining());
+ String s2 = new String(buf.array(), buf.position(),
+ buf.remaining(), UTF_8);
+ Assert.assertEquals(s, s2);
+ }
+
+ private static void serialize(ValueWriter cos, JsonParser p,
+ ByteArrayOutputStream os)
+ throws JsonParseException, IOException {
+ boolean[] isArray = new boolean[100];
+ int[] counts = new int[100];
+ int stackTop = -1;
+
+ while (p.nextToken() != null) {
+ switch (p.getCurrentToken()) {
+ case END_ARRAY:
+ Assert.assertTrue(isArray[stackTop]);
+ cos.writeArrayEnd();
+ stackTop--;
+ break;
+ case END_OBJECT:
+ Assert.assertFalse(isArray[stackTop]);
+ cos.writeMapEnd();
+ stackTop--;
+ break;
+ case START_ARRAY:
+ if (stackTop >= 0 && isArray[stackTop]) {
+ cos.setItemCount(1);
+ cos.startItem();
+ counts[stackTop]++;
+ }
+ cos.writeArrayStart();
+ isArray[++stackTop] = true;
+ counts[stackTop] = 0;
+ continue;
+ case VALUE_STRING:
+ if (stackTop >= 0 && isArray[stackTop]) {
+ cos.setItemCount(1);
+ cos.startItem();
+ counts[stackTop]++;
+ }
+ byte[] bb = p.getText().getBytes(UTF_8);
+ cos.writeBytes(bb);
+ break;
+ case START_OBJECT:
+ if (stackTop >= 0 && isArray[stackTop]) {
+ cos.setItemCount(1);
+ cos.startItem();
+ counts[stackTop]++;
+ }
+ cos.writeMapStart();
+ isArray[++stackTop] = false;
+ counts[stackTop] = 0;
+ continue;
+ case FIELD_NAME:
+ cos.setItemCount(1);
+ cos.startItem();
+ counts[stackTop]++;
+ cos.writeBytes(p.getCurrentName().getBytes(UTF_8));
+ break;
+ default:
+ throw new RuntimeException("Unsupported: " + p.getCurrentToken());
+ }
+ }
+ }
+
+ @DataProvider
+ public static Object[][] data() {
+ return new Object[][] {
+ { 64, 0, "" },
+ { 64, 0, jss(0, 'a') },
+ { 64, 0, jss(3, 'a') },
+ { 64, 0, jss(64, 'a') },
+ { 64, 0, jss(65, 'a') },
+ { 64, 0, jss(100, 'a') },
+ { 64, 1, "[]" },
+ { 64, 1, "[" + jss(0, 'a') + "]" },
+ { 64, 1, "[" + jss(3, 'a') + "]" },
+ { 64, 1, "[" + jss(61, 'a') + "]" },
+ { 64, 1, "[" + jss(62, 'a') + "]" },
+ { 64, 1, "[" + jss(64, 'a') + "]" },
+ { 64, 1, "[" + jss(65, 'a') + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(0, '0') + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(10, '0') + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(63, '0') + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(64, '0') + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(65, '0') + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(0, '0') + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(10, '0') + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(51, '0') + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(52, '0') + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(54, '0') + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(55, '0') + "]" },
+
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(0, '0')
+ + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(63, '0')
+ + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(64, '0')
+ + "]" },
+ { 64, 1, "[" + jss(0, 'a') + "," + jss(0, 'a') + "," + jss(65, '0')
+ + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(10, '0')
+ + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(23, '0')
+ + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(24, '0')
+ + "]" },
+ { 64, 1, "[" + jss(10, 'a') + "," + jss(20, 'A') + "," + jss(25, '0')
+ + "]" },
+ { 64, 2, "[[]]"},
+ { 64, 2, "[[" + jss(0, 'a') + "], []]" },
+ { 64, 2, "[[" + jss(10, 'a') + "], []]" },
+ { 64, 2, "[[" + jss(59, 'a') + "], []]" },
+ { 64, 2, "[[" + jss(60, 'a') + "], []]" },
+ { 64, 2, "[[" + jss(100, 'a') + "], []]" },
+ { 64, 2, "[[" + jss(10, '0') + ", " + jss(53, 'a') + "], []]" },
+ { 64, 2, "[[" + jss(10, '0') + ", " + jss(54, 'a') + "], []]" },
+ { 64, 2, "[[" + jss(10, '0') + ", " + jss(55, 'a') + "], []]" },
+
+ { 64, 2, "[[], [" + jss(0, 'a') + "]]" },
+ { 64, 2, "[[], [" + jss(10, 'a') + "]]" },
+ { 64, 2, "[[], [" + jss(63, 'a') + "]]" },
+ { 64, 2, "[[], [" + jss(64, 'a') + "]]" },
+ { 64, 2, "[[], [" + jss(65, 'a') + "]]" },
+ { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(53, 'a') + "]]" },
+ { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(54, 'a') + "]]" },
+ { 64, 2, "[[], [" + jss(10, '0') + ", " + jss(55, 'a') + "]]" },
+
+ { 64, 2, "[[" + jss(10, '0') + "]]"},
+ { 64, 2, "[[" + jss(62, '0') + "]]"},
+ { 64, 2, "[[" + jss(63, '0') + "]]"},
+ { 64, 2, "[[" + jss(64, '0') + "]]"},
+ { 64, 2, "[[" + jss(10, 'a') + ", " + jss(10, '0') + "]]"},
+ { 64, 2, "[[" + jss(10, 'a') + ", " + jss(52, '0') + "]]"},
+ { 64, 2, "[[" + jss(10, 'a') + ", " + jss(53, '0') + "]]"},
+ { 64, 2, "[[" + jss(10, 'a') + ", " + jss(54, '0') + "]]"},
+ { 64, 3, "[[[" + jss(10, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(62, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(63, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(64, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(10, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(52, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + ", " + jss(53, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(54, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(10, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(52, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(53, '0') + "]]]"},
+ { 64, 3, "[[[" + jss(10, 'a') + "], [" + jss(54, '0') + "]]]"},
+
+ { 64, 2, "[[\"p\"], [\"mn\"]]"},
+ { 64, 2, "[[\"pqr\"], [\"mn\"]]"},
+ { 64, 2, "[[\"pqrstuvwxyz\"], [\"mn\"]]"},
+ { 64, 2, "[[\"abc\", \"pqrstuvwxyz\"], [\"mn\"]]"},
+ { 64, 2, "[[\"mn\"], [\"\"]]"},
+ { 64, 2, "[[\"mn\"], \"abc\"]"},
+ { 64, 2, "[[\"mn\"], \"abcdefghijk\"]"},
+ { 64, 2, "[[\"mn\"], \"pqr\", \"abc\"]"},
+ { 64, 2, "[[\"mn\"]]"},
+ { 64, 2, "[[\"p\"], [\"mnopqrstuvwx\"]]"},
+ { 64, 2, "[[\"pqr\"], [\"mnopqrstuvwx\"]]"},
+ { 64, 2, "[[\"pqrstuvwxyz\"], [\"mnopqrstuvwx\"]]"},
+ { 64, 2, "[[\"abc\"], \"pqrstuvwxyz\", [\"mnopqrstuvwx\"]]"},
+ { 64, 2, "[[\"mnopqrstuvwx\"], [\"\"]]"},
+ { 64, 2, "[[\"mnopqrstuvwx\"], [\"abc\"]]"},
+ { 64, 2, "[[\"mnopqrstuvwx\"], [\"abcdefghijk\"]]"},
+ { 64, 2, "[[\"mnopqrstuvwx\"], [\"pqr\", \"abc\"]]"},
+ { 100, 2, "[[\"pqr\", \"mnopqrstuvwx\"]]"},
+ { 100, 2, "[[\"pqr\", \"ab\", \"mnopqrstuvwx\"]]"},
+ { 64, 2, "[[[\"pqr\"]], [[\"ab\"], [\"mnopqrstuvwx\"]]]"},
+
+ { 64, 1, "{}" },
+ { 64, 1, "{\"n\": \"v\"}" },
+ { 64, 1, "{\"n1\": \"v\", \"n2\": []}" },
+ { 100, 1, "{\"n1\": \"v\", \"n2\": []}" },
+ { 100, 1, "{\"n1\": \"v\", \"n2\": [\"abc\"]}" },
+ };
+ }
+
+ /**
+ * Returns a new JSON String {@code n} bytes long with
+ * consecutive characters starting with {@code c}.
+ */
+ private static String jss(final int n, char c) {
+ char[] cc = new char[n + 2];
+ cc[0] = cc[n + 1] = '"';
+ for (int i = 1; i < n + 1; i++) {
+ if (c == 'Z') {
+ c = 'a';
+ } else if (c == 'z') {
+ c = '0';
+ } else if (c == '9') {
+ c = 'A';
+ } else {
+ c++;
+ }
+ cc[i] = c;
+ }
+ return new String(cc);
+ }
+}
Modified: hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java?rev=782092&r1=782091&r2=782092&view=diff
==============================================================================
--- hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java (original)
+++ hadoop/avro/trunk/src/test/java/org/apache/avro/io/TestValueReader.java Fri Jun 5 18:31:45 2009
@@ -30,46 +30,51 @@
public class TestValueReader {
/** Verify EOFException throw at EOF */
- @Test
- public void testEOFHandling() throws IOException {
- InputStream is = new ByteArrayInputStream(new byte[0]);
- ValueReader vr = new ValueReader(is);
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_boolean() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).readBoolean();
+ }
+
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_int() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).readInt();
+ }
+
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_long() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).readLong();
+ }
+
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_float() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).readFloat();
+ }
+
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_double() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).readDouble();
+ }
+
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_bytes() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).readBytes(null);
+ }
+
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_string() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).
+ readString(new Utf8("a"));
+ }
+
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_fixed() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).
+ readFixed(new byte[1]);
+ }
- try {
- vr.readBoolean();
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readBuffer(null);
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readBytes(new byte[1]);
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readBytes(new byte[1], 0, 1);
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readDouble();
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readFloat();
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readInt();
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readLong();
- fail();
- } catch (EOFException e) { /* this is good */ }
- try {
- vr.readUtf8(new Utf8("a"));
- fail();
- } catch (EOFException e) { /* this is good */ }
+ @Test(expectedExceptions=EOFException.class)
+ public void testEOF_enum() throws IOException {
+ new ValueReader(new ByteArrayInputStream(new byte[0])).readEnum();
}
+
}