You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by ph...@apache.org on 2010/09/20 19:10:19 UTC
svn commit: r999010 - in /avro/trunk: CHANGES.txt
lang/java/src/java/org/apache/avro/tool/DataFileWriteTool.java
lang/java/src/test/bin/test_tools.sh
lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Author: philz
Date: Mon Sep 20 17:10:18 2010
New Revision: 999010
URL: http://svn.apache.org/viewvc?rev=999010&view=rev
Log:
AVRO-670. Allow DataFileWriteTool to accept schema files as input with new
--schema-file and --schema command-line flags. (Contribyted by Ron Bodkin)
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/src/java/org/apache/avro/tool/DataFileWriteTool.java
avro/trunk/lang/java/src/test/bin/test_tools.sh
avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=999010&r1=999009&r2=999010&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Mon Sep 20 17:10:18 2010
@@ -4,6 +4,9 @@ Avro 1.5.0 (unreleased)
INCOMPATIBLE CHANGES
+ AVRO-670. Allow DataFileWriteTool to accept schema files as input with new
+ --schema-file and --schema command-line flags. (Ron Bodkin via philz)
+
Avro 1.4.1 (unreleased)
NEW FEATURES
Modified: avro/trunk/lang/java/src/java/org/apache/avro/tool/DataFileWriteTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/src/java/org/apache/avro/tool/DataFileWriteTool.java?rev=999010&r1=999009&r2=999010&view=diff
==============================================================================
--- avro/trunk/lang/java/src/java/org/apache/avro/tool/DataFileWriteTool.java (original)
+++ avro/trunk/lang/java/src/java/org/apache/avro/tool/DataFileWriteTool.java Mon Sep 20 17:10:18 2010
@@ -17,9 +17,11 @@
*/
package org.apache.avro.tool;
+import java.io.DataInputStream;
import java.io.EOFException;
+import java.io.FileReader;
+import java.io.IOException;
import java.io.InputStream;
-import java.io.DataInputStream;
import java.io.PrintStream;
import java.util.List;
@@ -30,9 +32,9 @@ import joptsimple.OptionSpec;
import org.apache.avro.Schema;
import org.apache.avro.file.CodecFactory;
import org.apache.avro.file.DataFileWriter;
-import org.apache.avro.io.DatumReader;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
+import org.apache.avro.io.DatumReader;
import org.apache.avro.io.Decoder;
import org.apache.avro.io.JsonDecoder;
@@ -59,18 +61,37 @@ public class DataFileWriteTool implement
.withRequiredArg()
.defaultsTo("null")
.ofType(String.class);
+ OptionSpec<String> file =
+ p.accepts("schema-file", "Schema File")
+ .withOptionalArg()
+ .ofType(String.class);
+ OptionSpec<String> inschema =
+ p.accepts("schema", "Schema")
+ .withOptionalArg()
+ .ofType(String.class);
OptionSet opts = p.parse(args.toArray(new String[0]));
- if (opts.nonOptionArguments().size() != 2) {
- err.println("Expected 2 args: schema input_file");
+ List<String> nargs = opts.nonOptionArguments();
+ if (nargs.size() != 1) {
+ err.println("Expected 1 arg: input_file");
p.printHelpOn(err);
return 1;
}
-
- Schema schema = Schema.parse(args.get(0));
+ String schemastr = inschema.value(opts);
+ String schemafile = file.value(opts);
+ if (schemastr == null && schemafile == null) {
+ err.println("Need an input schema file (--schema-file) or inline schema (--schema)");
+ p.printHelpOn(err);
+ return 1;
+ }
+ if (schemafile != null) {
+ schemastr = readSchemaFromFile(schemafile);
+ }
+
+ Schema schema = Schema.parse(schemastr);
DatumReader<Object> reader = new GenericDatumReader<Object>(schema);
- InputStream input = Util.fileOrStdin(args.get(1), stdin);
+ InputStream input = Util.fileOrStdin(nargs.get(0), stdin);
try {
DataInputStream din = new DataInputStream(input);
DataFileWriter<Object> writer =
@@ -95,4 +116,22 @@ public class DataFileWriteTool implement
}
return 0;
}
+
+ public static String readSchemaFromFile(String schemafile) throws IOException {
+ String schemastr;
+ StringBuilder b = new StringBuilder();
+ FileReader r = new FileReader(schemafile);
+ try {
+ char[] buf = new char[64*1024];
+ for(;;) {
+ int read = r.read(buf);
+ if (read==-1) break;
+ b.append(buf, 0, read);
+ }
+ schemastr = b.toString();
+ } finally {
+ r.close();
+ }
+ return schemastr;
+ }
}
Modified: avro/trunk/lang/java/src/test/bin/test_tools.sh
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/src/test/bin/test_tools.sh?rev=999010&r1=999009&r2=999010&view=diff
==============================================================================
--- avro/trunk/lang/java/src/test/bin/test_tools.sh (original)
+++ avro/trunk/lang/java/src/test/bin/test_tools.sh Mon Sep 20 17:10:18 2010
@@ -73,7 +73,7 @@ echo '"The identity function"' \
| $CMD fragtojson '"string"' - \
| cmp -s - <(echo '"The identity function"')
-$CMD fromjson '"string"' <(echo '"foo"'; echo '"bar"') \
+$CMD fromjson --schema '"string"' <(echo '"foo"'; echo '"bar"') \
> $TMPDIR/data_file_write.avro
$CMD tojson $TMPDIR/data_file_write.avro \
| cmp -s - <(echo '"foo"'; echo '"bar"')
Modified: avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=999010&r1=999009&r2=999010&view=diff
==============================================================================
--- avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java (original)
+++ avro/trunk/lang/java/src/test/java/org/apache/avro/tool/TestDataFileTools.java Mon Sep 20 17:10:18 2010
@@ -25,6 +25,7 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.io.FileWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -46,12 +47,17 @@ public class TestDataFileTools {
static File sampleFile;
static String jsonData;
static Schema schema;
+ static File schemaFile;
@BeforeClass
public static void writeSampleFile() throws IOException {
sampleFile = AvroTestUtil.tempFile(
TestDataFileTools.class.getName() + ".avro");
schema = Schema.create(Type.INT);
+ schemaFile = File.createTempFile("schema-temp", "schema");
+ FileWriter fw = new FileWriter(schemaFile);
+ fw.append(schema.toString());
+ fw.close();
DataFileWriter<Object> writer
= new DataFileWriter<Object>(new GenericDatumWriter<Object>(schema))
@@ -108,12 +114,19 @@ public class TestDataFileTools {
public void testWrite(String name, List<String> extra, String expectedCodec)
throws Exception {
+ testWrite(name, extra, expectedCodec, "-schema", schema.toString());
+ testWrite(name, extra, expectedCodec, "-schema-file", schemaFile.toString());
+ }
+ public void testWrite(String name, List<String> extra, String expectedCodec, String... extraArgs)
+ throws Exception {
File outFile = AvroTestUtil.tempFile(
TestDataFileTools.class + ".testWrite." + name + ".avro");
FileOutputStream fout = new FileOutputStream(outFile);
PrintStream out = new PrintStream(fout);
List<String> args = new ArrayList<String>();
- args.add(schema.toString());
+ for (String arg : extraArgs) {
+ args.add(arg);
+ }
args.add("-");
args.addAll(extra);
new DataFileWriteTool().run(
@@ -149,8 +162,8 @@ public class TestDataFileTools {
new DataFileWriteTool().run(
new StringInputStream("{"),
new PrintStream(out), // stdout
- null, // stderr
- Arrays.asList("{ \"type\":\"record\", \"fields\":" +
+ null, // stderr
+ Arrays.asList("-schema", "{ \"type\":\"record\", \"fields\":" +
"[{\"name\":\"foo\", \"type\":\"string\"}], " +
"\"name\":\"boring\" }", "-"));
fail("Expected exception.");
@@ -196,7 +209,7 @@ public class TestDataFileTools {
new StringInputStream(json),
new PrintStream(out), // stdout
null, // stderr
- Arrays.asList(schema, "-"));
+ Arrays.asList("-schema", schema, "-"));
out.close();
fout.close();
return outFile;