You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2013/12/14 00:32:57 UTC
svn commit: r1550850 - in /avro/trunk: ./
lang/java/tools/src/main/java/org/apache/avro/tool/
lang/java/tools/src/test/java/org/apache/avro/tool/
Author: cutting
Date: Fri Dec 13 23:32:56 2013
New Revision: 1550850
URL: http://svn.apache.org/r1550850
Log:
AVRO-1397. Java: Binary fragment tools can now read multiple objects from their input. Contributed by Rob Turner.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Dec 13 23:32:56 2013
@@ -15,6 +15,9 @@ Trunk (not yet released)
AVRO-1400. Java: Add AvroDefault reflect annotation to specify
default values. (cutting)
+ AVRO-1397. Java: Binary fragment tools can now read multiple
+ objects from their input. (Rob Turner via cutting)
+
IMPROVEMENTS
AVRO-1355. Java: Reject schemas with duplicate field
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java Fri Dec 13 23:32:56 2013
@@ -22,15 +22,14 @@ import java.io.PrintStream;
import java.util.List;
import org.apache.avro.Schema;
+import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.DecoderFactory;
import org.apache.avro.io.DatumReader;
import org.apache.avro.io.DatumWriter;
import org.apache.avro.io.EncoderFactory;
+import org.apache.avro.io.JsonEncoder;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
-import org.codehaus.jackson.JsonEncoding;
-import org.codehaus.jackson.JsonFactory;
-import org.codehaus.jackson.JsonGenerator;
/** Converts an input file from Avro binary into JSON. */
public class BinaryFragmentToJsonTool implements Tool {
@@ -47,14 +46,16 @@ public class BinaryFragmentToJsonTool im
try {
DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
- Object datum = reader.read(null,
- DecoderFactory.get().binaryDecoder(input, null));
+ BinaryDecoder binaryDecoder =
+ DecoderFactory.get().binaryDecoder(input, null);
DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
- JsonGenerator g =
- new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
- g.useDefaultPrettyPrinter();
- writer.write(datum, EncoderFactory.get().jsonEncoder(schema, g));
- g.flush();
+ JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, out);
+ Object datum = null;
+ while (!binaryDecoder.isEnd()){
+ datum = reader.read(datum, binaryDecoder);
+ writer.write(datum, jsonEncoder);
+ jsonEncoder.flush();
+ }
out.println();
out.flush();
} finally {
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java Fri Dec 13 23:32:56 2013
@@ -17,6 +17,7 @@
*/
package org.apache.avro.tool;
+import java.io.EOFException;
import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
@@ -27,6 +28,7 @@ import org.apache.avro.generic.GenericDa
import org.apache.avro.io.Encoder;
import org.apache.avro.io.EncoderFactory;
import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.JsonDecoder;
/** Tool to convert JSON data into the binary form. */
public class JsonToBinaryFragmentTool implements Tool {
@@ -43,14 +45,20 @@ public class JsonToBinaryFragmentTool im
try {
GenericDatumReader<Object> reader =
new GenericDatumReader<Object>(schema);
- Object datum = reader.read(null,
- DecoderFactory.get().jsonDecoder(schema, input));
+ JsonDecoder jsonDecoder =
+ DecoderFactory.get().jsonDecoder(schema, input);
GenericDatumWriter<Object> writer =
new GenericDatumWriter<Object>(schema);
Encoder e = EncoderFactory.get().binaryEncoder(out, null);
- writer.write(datum, e);
- e.flush();
+ Object datum = null;
+ try {
+ while(true) {
+ datum = reader.read(datum, jsonDecoder);
+ writer.write(datum, e);
+ e.flush();
+ }
+ } catch (EOFException eofException) {}
} finally {
Util.close(input);
}
Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java?rev=1550850&r1=1550849&r2=1550850&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java Fri Dec 13 23:32:56 2013
@@ -26,6 +26,7 @@ import java.io.PrintStream;
import java.util.Arrays;
import org.apache.avro.Schema;
+import org.apache.avro.Schema.Type;
import org.junit.Test;
/**
@@ -33,35 +34,54 @@ import org.junit.Test;
* and {@link BinaryFragmentToJsonTool}.
*/
public class TestJsonToFromBinaryFragmentTools {
- private static final Schema STRING_SCHEMA = Schema.parse("\"string\"");
+ private static final Schema STRING_SCHEMA = Schema.create(Type.STRING);
private static final String UTF8 = "utf-8";
private static final String AVRO =
"ZLong string implies readable length encoding.";
private static final String JSON =
- "\"Long string implies readable length encoding.\"";
+ "\"Long string implies readable length encoding.\"\n";
@Test
public void testBinaryToJson() throws Exception {
+ binaryToJson(AVRO, JSON);
+ }
+
+ @Test
+ public void testJsonToBinary() throws Exception {
+ jsonToBinary(JSON, AVRO);
+ }
+
+ @Test
+ public void testMultiBinaryToJson() throws Exception {
+ binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON);
+ }
+
+ @Test
+ public void testMultiJsonToBinary() throws Exception {
+ jsonToBinary(JSON + JSON + JSON, AVRO + AVRO + AVRO);
+ }
+
+ private void binaryToJson(String avro, String json) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream p = new PrintStream(new BufferedOutputStream(baos));
new BinaryFragmentToJsonTool().run(
- new ByteArrayInputStream(AVRO.getBytes(UTF8)), // stdin
+ new ByteArrayInputStream(avro.getBytes(UTF8)), // stdin
p, // stdout
null, // stderr
Arrays.asList(STRING_SCHEMA.toString(), "-"));
- assertEquals(JSON + "\n", baos.toString(UTF8).replace("\r", ""));
+ assertEquals(json, baos.toString(UTF8).replace("\r", ""));
}
- @Test
- public void testJsonToBinary() throws Exception {
+ private void jsonToBinary(String json, String avro) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream p = new PrintStream(new BufferedOutputStream(baos));
+
new JsonToBinaryFragmentTool().run(
- new ByteArrayInputStream(JSON.getBytes(UTF8)), // stdin
+ new ByteArrayInputStream(json.getBytes(UTF8)), // stdin
p, // stdout
null, // stderr
Arrays.asList(STRING_SCHEMA.toString(), "-"));
- assertEquals(AVRO, baos.toString(UTF8));
+ assertEquals(avro, baos.toString(UTF8));
}
}