You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by th...@apache.org on 2010/01/14 14:40:26 UTC
svn commit: r899195 - in /hadoop/avro/trunk: ./
lang/java/src/java/org/apache/avro/io/
lang/java/src/java/org/apache/avro/io/parsing/
lang/java/src/test/java/org/apache/avro/io/
Author: thiru
Date: Thu Jan 14 13:40:25 2010
New Revision: 899195
URL: http://svn.apache.org/viewvc?rev=899195&view=rev
Log:
AVRO-316. Optiminzing inner loop functions of Avro io
Modified:
hadoop/avro/trunk/CHANGES.txt
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java
hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java
Modified: hadoop/avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/CHANGES.txt?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/CHANGES.txt (original)
+++ hadoop/avro/trunk/CHANGES.txt Thu Jan 14 13:40:25 2010
@@ -224,6 +224,8 @@
AVRO-315. Performance improvements to BinaryDecoder (thiru)
+ AVRO-316. Optiminzing inner loop functions of Avro io (thiru)
+
BUG FIXES
AVRO-176. Safeguard against bad istreams before reading. (sbanacho)
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonDecoder.java Thu Jan 14 13:40:25 2010
@@ -389,7 +389,7 @@
String fn = in.getCurrentName();
if (fa.fname.equals(fn)) {
in.nextToken();
- return Symbol.CONTINUE;
+ return null;
} else {
throw new AvroTypeException("Expected field name " + fa.fname +
" got " + in.getCurrentName());
@@ -410,7 +410,7 @@
} else {
throw new AvroTypeException("Unknown action symbol " + top);
}
- return Symbol.CONTINUE;
+ return null;
}
private AvroTypeException error(String type) {
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/JsonEncoder.java Thu Jan 14 13:40:25 2010
@@ -233,7 +233,7 @@
} else {
throw new AvroTypeException("Unknown action symbol " + top);
}
- return Symbol.CONTINUE;
+ return null;
}
}
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ResolvingDecoder.java Thu Jan 14 13:40:25 2010
@@ -157,7 +157,7 @@
@Override
public Symbol doAction(Symbol input, Symbol top) throws IOException {
if (top instanceof Symbol.FieldAdjustAction) {
- return input == Symbol.FIELD_ACTION ? top : Symbol.CONTINUE;
+ return input == Symbol.FIELD_ACTION ? top : null;
} if (top instanceof Symbol.ResolvingAction) {
Symbol.ResolvingAction t = (Symbol.ResolvingAction) top;
if (t.reader != input) {
@@ -183,7 +183,7 @@
} else {
throw new AvroTypeException("Unknown action: " + top);
}
- return Symbol.CONTINUE;
+ return null;
}
@Override
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingDecoder.java Thu Jan 14 13:40:25 2010
@@ -226,7 +226,7 @@
}
public Symbol doAction(Symbol input, Symbol top) throws IOException {
- return Symbol.CONTINUE;
+ return null;
}
}
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/ValidatingEncoder.java Thu Jan 14 13:40:25 2010
@@ -192,7 +192,7 @@
@Override
public Symbol doAction(Symbol input, Symbol top) throws IOException {
- return Symbol.CONTINUE;
+ return null;
}
/** Have we written at least one item into the current collection? */
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/JsonGrammarGenerator.java Thu Jan 14 13:40:25 2010
@@ -64,13 +64,11 @@
return Symbol.seq(new Symbol.EnumLabelsAction(sc.getEnumSymbols()),
Symbol.ENUM);
case ARRAY:
- return Symbol.seq(Symbol.ARRAY_END,
- Symbol.repeat(Symbol.ARRAY_END,
+ return Symbol.seq(Symbol.repeat(Symbol.ARRAY_END,
Symbol.ITEM_END, generate(sc.getElementType(), seen)),
Symbol.ARRAY_START);
case MAP:
- return Symbol.seq(Symbol.MAP_END,
- Symbol.repeat(Symbol.MAP_END,
+ return Symbol.seq(Symbol.repeat(Symbol.MAP_END,
Symbol.ITEM_END, generate(sc.getValueType(), seen),
Symbol.MAP_KEY_MARKER, Symbol.STRING),
Symbol.MAP_START);
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Parser.java Thu Jan 14 13:40:25 2010
@@ -35,6 +35,16 @@
* provide this help.
*/
public interface ActionHandler {
+ /**
+ * Handle the action symbol <tt>top</tt> when the <tt>input</tt> is
+ * sought to be taken off the stack.
+ * @param input The input symbol from the caller of advance
+ * @param top The symbol at the top the stack.
+ * @return <tt>null</tt> if advance() is to continue processing the
+ * stack. If not <tt>null</tt> the return value will be returned
+ * by advance().
+ * @throws IOException
+ */
Symbol doAction(Symbol input, Symbol top) throws IOException;
}
@@ -70,21 +80,25 @@
public final Symbol advance(Symbol input) throws IOException {
for (; ;) {
Symbol top = stack[--pos];
- if (top.kind == Symbol.Kind.TERMINAL) {
- if (top == input) {
- return top; // A common case
- } else {
- throw new AvroTypeException("Attempt to process a "
- + input + " when a "
- + top + " was expected.");
+ if (top == input) {
+ return top; // A common case
+ }
+
+ Symbol.Kind k = top.kind;
+ if (k == Symbol.Kind.IMPLICIT_ACTION) {
+ Symbol result = symbolHandler.doAction(input, top);
+ if (result != null) {
+ return result;
}
- } else if (top.kind == Symbol.Kind.IMPLICIT_ACTION) {
- Symbol result = symbolHandler.doAction(input, top);
- if (result != Symbol.CONTINUE) {
- return result;
- }
+ } else if (k == Symbol.Kind.TERMINAL) {
+ throw new AvroTypeException("Attempt to process a "
+ + input + " when a "
+ + top + " was expected.");
+ } else if (k == Symbol.Kind.REPEATER
+ && input == ((Symbol.Repeater) top).end) {
+ return input;
} else {
- pushProduction(input, top);
+ pushProduction(top);
}
}
}
@@ -112,16 +126,13 @@
* @param input
* @param sym
*/
- public final void pushProduction(Symbol input, Symbol sym) {
- if (sym.kind != Symbol.Kind.REPEATER ||
- input != ((Symbol.Repeater) sym).end) {
- Symbol[] p = sym.production;
- while (pos + p.length > stack.length) {
- expandStack();
- }
- System.arraycopy(p, 0, stack, pos, p.length);
- pos += p.length;
+ public final void pushProduction(Symbol sym) {
+ Symbol[] p = sym.production;
+ while (pos + p.length > stack.length) {
+ expandStack();
}
+ System.arraycopy(p, 0, stack, pos, p.length);
+ pos += p.length;
}
/**
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ResolvingGrammarGenerator.java Thu Jan 14 13:40:25 2010
@@ -98,15 +98,13 @@
break;
case ARRAY:
- return Symbol.seq(Symbol.ARRAY_END,
- Symbol.repeat(Symbol.ARRAY_END,
+ return Symbol.seq(Symbol.repeat(Symbol.ARRAY_END,
generate(writer.getElementType(),
reader.getElementType(), seen)),
Symbol.ARRAY_START);
case MAP:
- return Symbol.seq(Symbol.MAP_END,
- Symbol.repeat(Symbol.MAP_END,
+ return Symbol.seq(Symbol.repeat(Symbol.MAP_END,
generate(writer.getValueType(),
reader.getValueType(), seen), Symbol.STRING),
Symbol.MAP_START);
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/SkipParser.java Thu Jan 14 13:40:25 2010
@@ -64,7 +64,7 @@
skipHandler.skipAction();
} else {
--pos;
- pushProduction(null, top);
+ pushProduction(top);
}
continue outer;
}
@@ -79,7 +79,7 @@
int target = pos;
Symbol repeater = stack[--pos];
assert repeater.kind == Symbol.Kind.REPEATER;
- pushProduction(null, repeater);
+ pushProduction(repeater);
skipTo(target);
}
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/Symbol.java Thu Jan 14 13:40:25 2010
@@ -379,7 +379,6 @@
public static final Symbol ITEM_END = new Symbol.Terminal("item-end");
/* a pseudo terminal used by parsers */
- public static final Symbol CONTINUE = new Symbol.Terminal("continue");
public static final Symbol FIELD_ACTION =
new Symbol.Terminal("field-action");
Modified: hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java (original)
+++ hadoop/avro/trunk/lang/java/src/java/org/apache/avro/io/parsing/ValidatingGrammarGenerator.java Thu Jan 14 13:40:25 2010
@@ -71,12 +71,10 @@
return Symbol.seq(new Symbol.IntCheckAction(sc.getEnumSymbols().size()),
Symbol.ENUM);
case ARRAY:
- return Symbol.seq(Symbol.ARRAY_END,
- Symbol.repeat(Symbol.ARRAY_END, generate(sc.getElementType(), seen)),
+ return Symbol.seq(Symbol.repeat(Symbol.ARRAY_END, generate(sc.getElementType(), seen)),
Symbol.ARRAY_START);
case MAP:
- return Symbol.seq(Symbol.MAP_END,
- Symbol.repeat(Symbol.MAP_END,
+ return Symbol.seq(Symbol.repeat(Symbol.MAP_END,
generate(sc.getValueType(), seen), Symbol.STRING),
Symbol.MAP_START);
case RECORD: {
Modified: hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java
URL: http://svn.apache.org/viewvc/hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java?rev=899195&r1=899194&r2=899195&view=diff
==============================================================================
--- hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java (original)
+++ hadoop/avro/trunk/lang/java/src/test/java/org/apache/avro/io/Perf.java Thu Jan 14 13:40:25 2010
@@ -20,8 +20,13 @@
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
import java.util.Random;
+import junit.extensions.RepeatedTest;
+
import org.apache.avro.Schema;
/**
@@ -34,39 +39,45 @@
public static void main(String[] args) throws IOException {
- Test[] tests = null;
- if (args.length == 0) {
- tests = new Test[] { new ReadInt(),
- new ReadLong(), new ReadFloat(), new ReadDouble() };
- } else if (args.length == 1) {
- if (args[0].equals("-i")) {
- tests = new Test[] { new ReadInt() };
- } else if (args[0].equals("-f")) {
- tests = new Test[] { new ReadFloat() };
- } else if (args[0].equals("-d")) {
- tests = new Test[] { new ReadDouble() };
- } else if (args[0].equals("-l")) {
- tests = new Test[] { new ReadLong() };
- }
- } else {
- usage();
- System.exit(1);
+ List<Test> tests = new ArrayList<Test>();
+ for (String arg : args) {
+ if (arg.equals("-i")) {
+ tests.add(new ReadInt());
+ } else if (arg.equals("-f")) {
+ tests.add(new ReadFloat());
+ } else if (arg.equals("-d")) {
+ tests.add(new ReadDouble());
+ } else if (arg.equals("-l")) {
+ tests.add(new ReadLong());
+ } else if (arg.equals("-R")) {
+ tests.add(new RepeaterTest());
+ } else {
+ usage();
+ }
+ }
+ if (tests.isEmpty()) {
+ tests.addAll(Arrays.asList(new Test[] {
+ new ReadInt(), new ReadLong(), new ReadFloat(), new ReadDouble(),
+ new RepeaterTest(),
+ }));
}
for (Test t : tests) {
- // warmup JVM
+ // warmup JVM
+ // System.out.println("Warming up...");
for (int i = 0; i < CYCLES; i++) {
- t.read();
+ t.test();
}
// test
long s = 0;
for (int i = 0; i < CYCLES; i++) {
- long l = t.read();
+ long l = t.test();
// System.out.println("** " + l);
s += l;
}
- s /= 1000;
- System.out.println(t.name + "(" + t.schema + "): " + s/1000 + " ms, " + (CYCLES * (double)COUNT)/s + " million numbers decoded /sec" );
+ s /= 1000;
+ System.out.println(t.name + "(" + t.schema + "): " + s/1000 + " ms, "
+ + ((CYCLES * (double)COUNT) / s) + " million numbers decoded/sec" );
}
}
@@ -74,6 +85,7 @@
public final String name;
public final Schema schema;
protected byte[] data;
+
public Test(String name, String json) throws IOException {
this.name = name;
this.schema = Schema.parse(json);
@@ -82,8 +94,9 @@
genData(e);
data = bao.toByteArray();
}
- public final long read() throws IOException {
- Decoder d = new BinaryDecoder(new ByteArrayInputStream(data));
+
+ public final long test() throws IOException {
+ Decoder d = getDecoder();
long t = System.nanoTime();
for (long l = d.readArrayStart(); l > 0; l = d.arrayNext()) {
for (int j = 0; j < l; j++) {
@@ -92,13 +105,22 @@
}
return (System.nanoTime() - t);
}
+
+ protected Decoder getDecoder() throws IOException {
+ return new BinaryDecoder(new ByteArrayInputStream(data));
+ }
+
abstract void genData(Encoder e) throws IOException;
abstract void readInternal(Decoder d) throws IOException;
}
private static class ReadInt extends Test {
public ReadInt() throws IOException {
- super("ReadInt", "{ \"type\": \"array\", \"items\": \"int\"} ");
+ this("ReadInt", "{ \"type\": \"array\", \"items\": \"int\"} ");
+ }
+
+ public ReadInt(String name, String json) throws IOException {
+ super(name, json);
}
@Override void genData(Encoder e) throws IOException {
e.writeArrayStart();
@@ -179,11 +201,26 @@
d.readDouble();
}
}
+
+ private static class RepeaterTest extends ReadInt {
+ public RepeaterTest() throws IOException {
+ super("RepeaterTest",
+ "{ \"type\": \"array\", \"items\": \"int\"} ");
+ }
+
+ @Override
+ public Decoder getDecoder() throws IOException {
+ return new ValidatingDecoder(schema, super.getDecoder());
+ }
+ }
+
private static void usage() {
System.out.println("Usage: Perf { -i | -l | -f | -d }");
System.out.println(" -i measures readInt() performance");
System.out.println(" -l measures readLong() performance");
System.out.println(" -f measures readFloat() performance");
System.out.println(" -d measures readDouble() performance");
+ System.out.println(" -R measures repeater performance in "
+ + "validating encoder");
}
}