You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2013/12/16 21:30:38 UTC
svn commit: r1551338 - in /avro/trunk: ./
lang/java/avro/src/main/java/org/apache/avro/io/
lang/java/tools/src/main/java/org/apache/avro/tool/
lang/java/tools/src/test/java/org/apache/avro/tool/
Author: cutting
Date: Mon Dec 16 20:30:38 2013
New Revision: 1551338
URL: http://svn.apache.org/r1551338
Log:
AVRO-1396. Java: Enable tojson command-line tool to pretty print output. Contributed by Rob Turner.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java
avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Mon Dec 16 20:30:38 2013
@@ -18,6 +18,9 @@ Trunk (not yet released)
AVRO-1397. Java: Binary fragment tools can now read multiple
objects from their input. (Rob Turner via cutting)
+ AVRO-1396. Java: Enable tojson command-line tool to pretty print output.
+ (Rob Turner via cutting)
+
IMPROVEMENTS
AVRO-1355. Java: Reject schemas with duplicate field
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/EncoderFactory.java Mon Dec 16 20:30:38 2013
@@ -283,6 +283,29 @@ public class EncoderFactory {
}
/**
+ * Creates a {@link JsonEncoder} using the OutputStream provided for writing
+ * data conforming to the Schema provided with optional pretty printing.
+ * <p/>
+ * {@link JsonEncoder} buffers its output. Data may not appear on the
+ * underlying OutputStream until {@link Encoder#flush()} is called.
+ * <p/>
+ * {@link JsonEncoder} is not thread-safe.
+ *
+ * @param schema
+ * The Schema for data written to this JsonEncoder. Cannot be null.
+ * @param out
+ * The OutputStream to write to. Cannot be null.
+ * @param pretty
+ * Pretty print encoding.
+ * @return A JsonEncoder configured with <i>out</i>, <i>schema</i> and <i>pretty</i>
+ * @throws IOException
+ */
+ public JsonEncoder jsonEncoder(Schema schema, OutputStream out, boolean pretty)
+ throws IOException {
+ return new JsonEncoder(schema, out, pretty);
+ }
+
+ /**
* Creates a {@link JsonEncoder} using the {@link JsonGenerator} provided for
* output of data conforming to the Schema provided.
* <p/>
Modified: avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java (original)
+++ avro/trunk/lang/java/avro/src/main/java/org/apache/avro/io/JsonEncoder.java Mon Dec 16 20:30:38 2013
@@ -31,6 +31,7 @@ import org.apache.avro.util.Utf8;
import org.codehaus.jackson.JsonEncoding;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
+import org.codehaus.jackson.util.DefaultPrettyPrinter;
import org.codehaus.jackson.util.MinimalPrettyPrinter;
/** An {@link Encoder} for Avro's JSON data encoding.
@@ -43,6 +44,7 @@ import org.codehaus.jackson.util.Minimal
* JsonEncoder is not thread-safe.
* */
public class JsonEncoder extends ParsingEncoder implements Parser.ActionHandler {
+ private static final String LINE_SEPARATOR = System.getProperty("line.separator");
final Parser parser;
private JsonGenerator out;
/**
@@ -51,7 +53,11 @@ public class JsonEncoder extends Parsing
protected BitSet isEmpty = new BitSet();
JsonEncoder(Schema sc, OutputStream out) throws IOException {
- this(sc, getJsonGenerator(out));
+ this(sc, getJsonGenerator(out, false));
+ }
+
+ JsonEncoder(Schema sc, OutputStream out, boolean pretty) throws IOException {
+ this(sc, getJsonGenerator(out, pretty));
}
JsonEncoder(Schema sc, JsonGenerator out) throws IOException {
@@ -68,16 +74,29 @@ public class JsonEncoder extends Parsing
}
}
- // by default, one object per line
- private static JsonGenerator getJsonGenerator(OutputStream out)
+ // by default, one object per line.
+ // with pretty option use default pretty printer with root line separator.
+ private static JsonGenerator getJsonGenerator(OutputStream out, boolean pretty)
throws IOException {
if (null == out)
throw new NullPointerException("OutputStream cannot be null");
JsonGenerator g
= new JsonFactory().createJsonGenerator(out, JsonEncoding.UTF8);
- MinimalPrettyPrinter pp = new MinimalPrettyPrinter();
- pp.setRootValueSeparator(System.getProperty("line.separator"));
- g.setPrettyPrinter(pp);
+ if (pretty) {
+ DefaultPrettyPrinter pp = new DefaultPrettyPrinter() {
+ //@Override
+ public void writeRootValueSeparator(JsonGenerator jg)
+ throws IOException
+ {
+ jg.writeRaw(LINE_SEPARATOR);
+ }
+ };
+ g.setPrettyPrinter(pp);
+ } else {
+ MinimalPrettyPrinter pp = new MinimalPrettyPrinter();
+ pp.setRootValueSeparator(LINE_SEPARATOR);
+ g.setPrettyPrinter(pp);
+ }
return g;
}
@@ -96,7 +115,7 @@ public class JsonEncoder extends Parsing
* @return this JsonEncoder
*/
public JsonEncoder configure(OutputStream out) throws IOException {
- this.configure(getJsonGenerator(out));
+ this.configure(getJsonGenerator(out, false));
return this;
}
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/BinaryFragmentToJsonTool.java Mon Dec 16 20:30:38 2013
@@ -21,6 +21,10 @@ import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
import org.apache.avro.Schema;
import org.apache.avro.io.BinaryDecoder;
import org.apache.avro.io.DecoderFactory;
@@ -36,20 +40,43 @@ public class BinaryFragmentToJsonTool im
@Override
public int run(InputStream stdin, PrintStream out, PrintStream err,
List<String> args) throws Exception {
- if (args.size() != 2) {
- err.println("Expected 2 arguments: schema binary_data_file");
- err.println("Use '-' as binary_data_file for stdin.");
+ OptionParser optionParser = new OptionParser();
+ OptionSpec<Void> noPrettyOption = optionParser
+ .accepts("no-pretty", "Turns off pretty printing.");
+ OptionSpec<String> schemaFileOption = optionParser
+ .accepts("schema-file", "File containing schema, must not occur with inline schema.")
+ .withOptionalArg()
+ .ofType(String.class);
+
+ OptionSet optionSet = optionParser.parse(args.toArray(new String[0]));
+ Boolean noPretty = optionSet.has(noPrettyOption);
+ List<String> nargs = optionSet.nonOptionArguments();
+ String schemaFile = schemaFileOption.value(optionSet);
+
+ if (nargs.size() != (schemaFile == null ? 2 : 1)) {
+ err.println("fragtojson --no-pretty --schema-file <file> [inline-schema] input-file");
+ err.println(" converts Avro fragments to JSON.");
+ optionParser.printHelpOn(err);
+ err.println(" A dash '-' for input-file means stdin.");
return 1;
}
- Schema schema = new Schema.Parser().parse(args.get(0));
- InputStream input = Util.fileOrStdin(args.get(1), stdin);
+ Schema schema;
+ String inputFile;
+ if (schemaFile == null) {
+ schema = new Schema.Parser().parse(nargs.get(0));
+ inputFile = nargs.get(1);
+ } else {
+ schema = new Schema.Parser().parse(Util.openFromFS(schemaFile));
+ inputFile = nargs.get(0);
+ }
+ InputStream input = Util.fileOrStdin(inputFile, stdin);
try {
DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
BinaryDecoder binaryDecoder =
DecoderFactory.get().binaryDecoder(input, null);
DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
- JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, out);
+ JsonEncoder jsonEncoder = EncoderFactory.get().jsonEncoder(schema, out, !noPretty);
Object datum = null;
while (!binaryDecoder.isEnd()){
datum = reader.read(datum, binaryDecoder);
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/DataFileReadTool.java Mon Dec 16 20:30:38 2013
@@ -21,6 +21,10 @@ import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
import org.apache.avro.Schema;
import org.apache.avro.file.FileReader;
import org.apache.avro.file.DataFileReader;
@@ -40,26 +44,36 @@ public class DataFileReadTool implements
@Override
public String getShortDescription() {
- return "Dumps an Avro data file as JSON, one record per line.";
+ return "Dumps an Avro data file as JSON, record per line or pretty.";
}
@Override
public int run(InputStream stdin, PrintStream out, PrintStream err,
List<String> args) throws Exception {
- if (args.size() != 1) {
+ OptionParser optionParser = new OptionParser();
+ OptionSpec<Void> prettyOption = optionParser
+ .accepts("pretty", "Turns on pretty printing.");
+
+ OptionSet optionSet = optionParser.parse(args.toArray(new String[0]));
+ Boolean pretty = optionSet.has(prettyOption);
+ List<String> nargs = optionSet.nonOptionArguments();
+
+ if (nargs.size() != 1) {
// Unlike other commands, "-" can't be used for stdin, because
// we can only use seekable files.
- err.println("Expected 1 argument: input_file.");
+ err.println("tojson --pretty input-file");
+ err.println(" converts Avro data file to JSON.");
+ optionParser.printHelpOn(err);
return 1;
}
GenericDatumReader<Object> reader = new GenericDatumReader<Object>();
FileReader<Object> fileReader =
- DataFileReader.openReader(Util.openSeekableFromFS(args.get(0)), reader);
+ DataFileReader.openReader(Util.openSeekableFromFS(nargs.get(0)), reader);
try {
Schema schema = fileReader.getSchema();
DatumWriter<Object> writer = new GenericDatumWriter<Object>(schema);
- JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out);
+ JsonEncoder encoder = EncoderFactory.get().jsonEncoder(schema, out, pretty);
for (Object datum : fileReader)
writer.write(datum, encoder);
encoder.flush();
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/JsonToBinaryFragmentTool.java Mon Dec 16 20:30:38 2013
@@ -22,6 +22,10 @@ import java.io.InputStream;
import java.io.PrintStream;
import java.util.List;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
import org.apache.avro.Schema;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
@@ -35,30 +39,53 @@ public class JsonToBinaryFragmentTool im
@Override
public int run(InputStream stdin, PrintStream out, PrintStream err,
List<String> args) throws Exception {
- if (args.size() != 2) {
- err.println("Expected 2 arguments: schema json_data_file");
- err.println("Use '-' as json_data_file for stdin.");
+ OptionParser optionParser = new OptionParser();
+ OptionSpec<String> schemaFileOption = optionParser
+ .accepts("schema-file", "File containing schema, must not occur with inline schema.")
+ .withOptionalArg()
+ .ofType(String.class);
+
+ OptionSet optionSet = optionParser.parse(args.toArray(new String[0]));
+ List<String> nargs = optionSet.nonOptionArguments();
+ String schemaFile = schemaFileOption.value(optionSet);
+
+ if (nargs.size() != (schemaFile == null ? 2 : 1)) {
+ err.println("jsontofrag --schema-file <file> [inline-schema] input-file");
+ err.println(" converts JSON to Avro fragments.");
+ optionParser.printHelpOn(err);
+ err.println(" A dash '-' for input-file means stdin.");
return 1;
}
- Schema schema = new Schema.Parser().parse(args.get(0));
- InputStream input = Util.fileOrStdin(args.get(1), stdin);
+ Schema schema;
+ String inputFile;
+ if (schemaFile == null) {
+ schema = new Schema.Parser().parse(nargs.get(0));
+ inputFile = nargs.get(1);
+ } else {
+ schema = new Schema.Parser().parse(Util.openFromFS(schemaFile));
+ inputFile = nargs.get(0);
+ }
+ InputStream input = Util.fileOrStdin(inputFile, stdin);
+
try {
- GenericDatumReader<Object> reader =
- new GenericDatumReader<Object>(schema);
+ GenericDatumReader<Object> reader =
+ new GenericDatumReader<Object>(schema);
- JsonDecoder jsonDecoder =
+ JsonDecoder jsonDecoder =
DecoderFactory.get().jsonDecoder(schema, input);
- GenericDatumWriter<Object> writer =
- new GenericDatumWriter<Object>(schema);
- Encoder e = EncoderFactory.get().binaryEncoder(out, null);
- Object datum = null;
- try {
+ GenericDatumWriter<Object> writer =
+ new GenericDatumWriter<Object>(schema);
+ Encoder e = EncoderFactory.get().binaryEncoder(out, null);
+ Object datum = null;
while(true) {
- datum = reader.read(datum, jsonDecoder);
+ try {
+ datum = reader.read(datum, jsonDecoder);
+ } catch (EOFException eofException) {
+ break;
+ }
writer.write(datum, e);
e.flush();
}
- } catch (EOFException eofException) {}
} finally {
Util.close(input);
}
Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java Mon Dec 16 20:30:38 2013
@@ -100,6 +100,12 @@ public class TestDataFileTools {
}
@Test
+ public void testReadToJsonPretty() throws Exception {
+ assertEquals(jsonData.toString(),
+ run(new DataFileReadTool(), "--pretty", sampleFile.getPath()));
+ }
+
+ @Test
public void testGetMeta() throws Exception {
String output = run(new DataFileGetMetaTool(), sampleFile.getPath());
assertTrue(output, output.contains("avro.schema\t"+schema.toString()+"\n"));
Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java?rev=1551338&r1=1551337&r2=1551338&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestJsonToFromBinaryFragmentTools.java Mon Dec 16 20:30:38 2013
@@ -22,11 +22,18 @@ import static org.junit.Assert.assertEqu
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
+import org.apache.avro.AvroTestUtil;
import org.apache.avro.Schema;
import org.apache.avro.Schema.Type;
+import org.junit.BeforeClass;
import org.junit.Test;
/**
@@ -34,7 +41,7 @@ import org.junit.Test;
* and {@link BinaryFragmentToJsonTool}.
*/
public class TestJsonToFromBinaryFragmentTools {
- private static final Schema STRING_SCHEMA = Schema.create(Type.STRING);
+ private static final String STRING_SCHEMA = Schema.create(Type.STRING).toString();
private static final String UTF8 = "utf-8";
private static final String AVRO =
"ZLong string implies readable length encoding.";
@@ -43,45 +50,80 @@ public class TestJsonToFromBinaryFragmen
@Test
public void testBinaryToJson() throws Exception {
- binaryToJson(AVRO, JSON);
+ binaryToJson(AVRO, JSON, STRING_SCHEMA);
}
@Test
public void testJsonToBinary() throws Exception {
- jsonToBinary(JSON, AVRO);
+ jsonToBinary(JSON, AVRO, STRING_SCHEMA);
}
@Test
public void testMultiBinaryToJson() throws Exception {
- binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON);
+ binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON, STRING_SCHEMA);
}
@Test
public void testMultiJsonToBinary() throws Exception {
- jsonToBinary(JSON + JSON + JSON, AVRO + AVRO + AVRO);
+ jsonToBinary(JSON + JSON + JSON, AVRO + AVRO + AVRO, STRING_SCHEMA);
}
- private void binaryToJson(String avro, String json) throws Exception {
+ @Test
+ public void testBinaryToNoPrettyJson() throws Exception {
+ binaryToJson(AVRO, JSON, "--no-pretty", STRING_SCHEMA);
+ }
+
+ @Test
+ public void testMultiBinaryToNoPrettyJson() throws Exception {
+ binaryToJson(AVRO + AVRO + AVRO, JSON + JSON + JSON, "--no-pretty", STRING_SCHEMA);
+ }
+
+ @Test
+ public void testBinaryToJsonSchemaFile() throws Exception {
+ binaryToJson(AVRO, JSON, "--schema-file", schemaFile());
+ }
+
+ @Test
+ public void testJsonToBinarySchemaFile() throws Exception {
+ jsonToBinary(JSON, AVRO, "--schema-file", schemaFile());
+ }
+
+ private void binaryToJson(String avro, String json, String... options) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream p = new PrintStream(new BufferedOutputStream(baos));
+ List<String> args = new ArrayList<String>();
+ args.addAll(Arrays.asList(options));
+ args.add("-");
new BinaryFragmentToJsonTool().run(
new ByteArrayInputStream(avro.getBytes(UTF8)), // stdin
p, // stdout
null, // stderr
- Arrays.asList(STRING_SCHEMA.toString(), "-"));
+ args);
+ System.out.println(baos.toString(UTF8).replace("\r", ""));
assertEquals(json, baos.toString(UTF8).replace("\r", ""));
}
-
- private void jsonToBinary(String json, String avro) throws Exception {
+
+ private void jsonToBinary(String json, String avro, String... options) throws Exception {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintStream p = new PrintStream(new BufferedOutputStream(baos));
+ List<String> args = new ArrayList<String>();
+ args.addAll(Arrays.asList(options));
+ args.add("-");
new JsonToBinaryFragmentTool().run(
new ByteArrayInputStream(json.getBytes(UTF8)), // stdin
p, // stdout
null, // stderr
- Arrays.asList(STRING_SCHEMA.toString(), "-"));
+ args);
assertEquals(avro, baos.toString(UTF8));
}
+
+ private static String schemaFile() throws IOException {
+ File schemaFile = AvroTestUtil.tempFile(TestJsonToFromBinaryFragmentTools.class, "String.avsc");
+ FileWriter fw = new FileWriter(schemaFile);
+ fw.append(STRING_SCHEMA);
+ fw.close();
+ return schemaFile.toString();
+ }
}