You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2014/09/23 18:25:28 UTC
svn commit: r1627083 - in /avro/trunk: ./
lang/java/tools/src/main/java/org/apache/avro/tool/
lang/java/tools/src/test/compiler/input/
lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/
lang/java/tools/src/test/compiler/output/ lan...
Author: cutting
Date: Tue Sep 23 16:25:27 2014
New Revision: 1627083
URL: http://svn.apache.org/r1627083
Log:
AVRO-1551. Java: Add an output encoding option to the compiler. Contributed by Keegan Witt.
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java
avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc
avro/trunk/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java
avro/trunk/lang/java/tools/src/test/compiler/output/Player.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1627083&r1=1627082&r2=1627083&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Tue Sep 23 16:25:27 2014
@@ -23,6 +23,9 @@ Trunk (not yet released)
AVRO-1583. Java: Add stdin support to the tojson tool.
(Clément Mahtieu via cutting)
+ AVRO-1551. Java: Add an output encoding option to the compiler
+ command line tool. (Keegan Witt via cutting)
+
BUG FIXES
AVRO-1553. Java: MapReduce never uses MapOutputValueSchema (tomwhite)
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java?rev=1627083&r1=1627082&r2=1627083&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java Tue Sep 23 16:25:27 2014
@@ -42,7 +42,7 @@ public class SpecificCompilerTool implem
List<String> args) throws Exception {
if (args.size() < 3) {
System.err
- .println("Usage: [-string] (schema|protocol) input... outputdir");
+ .println("Usage: [-encoding <outputencoding>] [-string] (schema|protocol) input... outputdir");
System.err
.println(" input - input files or directories");
System.err
@@ -54,11 +54,19 @@ public class SpecificCompilerTool implem
StringType stringType = StringType.CharSequence;
int arg = 0;
+
+ String encoding = null;
+ if ("-encoding".equals(args.get(arg))) {
+ arg++;
+ encoding = args.get(arg);
+ arg++;
+ }
+
if ("-string".equals(args.get(arg))) {
stringType = StringType.String;
arg++;
}
-
+
String method = args.get(arg);
List<File> inputs = new ArrayList<File>();
File output = new File(args.get(args.size() - 1));
@@ -73,6 +81,9 @@ public class SpecificCompilerTool implem
Schema schema = parser.parse(src);
SpecificCompiler compiler = new SpecificCompiler(schema);
compiler.setStringType(stringType);
+ if (encoding != null) {
+ compiler.setOutputCharacterEncoding(encoding);
+ }
compiler.compileToDestination(src, output);
}
} else if ("protocol".equals(method)) {
@@ -80,6 +91,9 @@ public class SpecificCompilerTool implem
Protocol protocol = Protocol.parse(src);
SpecificCompiler compiler = new SpecificCompiler(protocol);
compiler.setStringType(stringType);
+ if (encoding != null) {
+ compiler.setOutputCharacterEncoding(encoding);
+ }
compiler.compileToDestination(src, output);
}
} else {
Modified: avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc?rev=1627083&r1=1627082&r2=1627083&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc (original)
+++ avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc Tue Sep 23 16:25:27 2014
@@ -1,4 +1,4 @@
-{"type":"record", "name":"Player", "namespace": "avro.examples.baseball",
+{"type":"record", "name":"Player", "namespace": "avro.examples.baseball","doc":"é¸æ is Japanese for player.",
"fields": [
{"name": "number", "type": "int"},
{"name": "first_name", "type": "string"},
Modified: avro/trunk/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java?rev=1627083&r1=1627082&r2=1627083&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java (original)
+++ avro/trunk/lang/java/tools/src/test/compiler/output-string/avro/examples/baseball/Player.java Tue Sep 23 16:25:27 2014
@@ -4,10 +4,10 @@
* DO NOT EDIT DIRECTLY
*/
package avro.examples.baseball;
-@SuppressWarnings("all")
+@SuppressWarnings("all")/** é¸æ is Japanese for player. */
@org.apache.avro.specific.AvroGenerated
public class Player extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Player\",\"namespace\":\"avro.examples.baseball\",\"fields\":[{\"name\":\"number\",\"type\":\"int\"},{\"name\":\"first_name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"last_name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"position\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"enum\",\"name\":\"Position\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}}}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Player\",\"namespace\":\"avro.examples.baseball\",\"doc\":\"é¸æ is Japanese for player.\",\"fields\":[{\"name\":\"number\",\"type\":\"int\"},{\"name\":\"first_name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"last_name\",\"type\":{\"type\":\"string\",\"avro.java.string\":\"String\"}},{\"name\":\"position\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"enum\",\"name\":\"Position\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}}}]}");
public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
@Deprecated public int number;
@Deprecated public java.lang.String first_name;
Modified: avro/trunk/lang/java/tools/src/test/compiler/output/Player.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/output/Player.java?rev=1627083&r1=1627082&r2=1627083&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/output/Player.java (original)
+++ avro/trunk/lang/java/tools/src/test/compiler/output/Player.java Tue Sep 23 16:25:27 2014
@@ -4,10 +4,10 @@
* DO NOT EDIT DIRECTLY
*/
package avro.examples.baseball;
-@SuppressWarnings("all")
+@SuppressWarnings("all")/** é¸æ is Japanese for player. */
@org.apache.avro.specific.AvroGenerated
public class Player extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
- public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Player\",\"namespace\":\"avro.examples.baseball\",\"fields\":[{\"name\":\"number\",\"type\":\"int\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"position\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"enum\",\"name\":\"Position\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}}}]}");
+ public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"Player\",\"namespace\":\"avro.examples.baseball\",\"doc\":\"é¸æ is Japanese for player.\",\"fields\":[{\"name\":\"number\",\"type\":\"int\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"position\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"enum\",\"name\":\"Position\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}}}]}");
public static org.apache.avro.Schema getClassSchema() { return SCHEMA$; }
@Deprecated public int number;
@Deprecated public java.lang.CharSequence first_name;
Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java?rev=1627083&r1=1627082&r2=1627083&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java Tue Sep 23 16:25:27 2014
@@ -23,8 +23,9 @@ import org.junit.Test;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileReader;
+import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStreamReader;
import java.util.Arrays;
/**
@@ -80,7 +81,7 @@ public class TestSpecificCompilerTool {
@Test
public void testCompileSchemaSingleFile() throws Exception {
- doCompile(new String[]{"schema",
+ doCompile(new String[]{"-encoding", "UTF-8", "schema",
TEST_INPUT_DIR.toString() + "/position.avsc",
TEST_OUTPUT_DIR.getPath()});
assertFileMatch(TEST_EXPECTED_POSITION, TEST_OUTPUT_POSITION);
@@ -89,7 +90,7 @@ public class TestSpecificCompilerTool {
@Test
public void testCompileSchemaTwoFiles() throws Exception {
- doCompile(new String[]{"schema",
+ doCompile(new String[]{"-encoding", "UTF-8", "schema",
TEST_INPUT_DIR.toString() + "/position.avsc",
TEST_INPUT_DIR.toString() + "/player.avsc",
TEST_OUTPUT_DIR.getPath()});
@@ -100,7 +101,7 @@ public class TestSpecificCompilerTool {
@Test
public void testCompileSchemaFileAndDirectory() throws Exception {
- doCompile(new String[]{"schema",
+ doCompile(new String[]{"-encoding", "UTF-8", "schema",
TEST_INPUT_DIR.toString() + "/position.avsc",
TEST_INPUT_DIR.toString(),
TEST_OUTPUT_DIR.getPath()});
@@ -111,7 +112,8 @@ public class TestSpecificCompilerTool {
@Test
public void testCompileSchemasUsingString() throws Exception {
- doCompile(new String[]{"-string", "schema",
+ doCompile(new String[]{"-encoding", "UTF-8",
+ "-string", "schema",
TEST_INPUT_DIR.toString() + "/position.avsc",
TEST_INPUT_DIR.toString() + "/player.avsc",
TEST_OUTPUT_STRING_DIR.getPath()});
@@ -145,7 +147,7 @@ public class TestSpecificCompilerTool {
* file content and comparing provides nice diffs via JUnit when failures occur.
*/
private static String readFile(File file) throws IOException {
- BufferedReader reader = new BufferedReader(new FileReader(file));
+ BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
StringBuilder sb = new StringBuilder();
String line = null;
boolean first = true;