You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by cu...@apache.org on 2011/08/25 22:51:43 UTC
svn commit: r1161755 - in /avro/trunk: ./
lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/
lang/java/tools/src/main/java/org/apache/avro/tool/
lang/java/tools/src/test/compiler/ lang/java/tools/src/test/compiler/input/
lang/java/tool...
Author: cutting
Date: Thu Aug 25 20:51:43 2011
New Revision: 1161755
URL: http://svn.apache.org/viewvc?rev=1161755&view=rev
Log:
AVRO-877. Java: Add support for compiling multiple, dependent schemas. Contributed by Bill Graham.
Added:
avro/trunk/lang/java/tools/src/test/compiler/
avro/trunk/lang/java/tools/src/test/compiler/input/
avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc
avro/trunk/lang/java/tools/src/test/compiler/input/position.avsc
avro/trunk/lang/java/tools/src/test/compiler/output/
avro/trunk/lang/java/tools/src/test/compiler/output/Player.java
avro/trunk/lang/java/tools/src/test/compiler/output/Position.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1161755&r1=1161754&r2=1161755&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Thu Aug 25 20:51:43 2011
@@ -51,6 +51,9 @@ Avro 1.5.3 (unreleased)
AVRO-874. Java: Improved Schema parsing API and permit IDL imports
to depend on names defined in prior imports. (cutting)
+ AVRO-877. Java: Add support for compiling multiple, dependent
+ schemas. (Bill Graham via cutting)
+
BUG FIXES
Avro 1.5.2 (12 August 2011)
Modified: avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java?rev=1161755&r1=1161754&r2=1161755&view=diff
==============================================================================
--- avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java (original)
+++ avro/trunk/lang/java/compiler/src/main/java/org/apache/avro/compiler/specific/SpecificCompiler.java Thu Aug 25 20:51:43 2011
@@ -153,16 +153,36 @@ public class SpecificCompiler {
* @param dest the directory to place generated files in
*/
public static void compileProtocol(File src, File dest) throws IOException {
- Protocol protocol = Protocol.parse(src);
- SpecificCompiler compiler = new SpecificCompiler(protocol);
- compiler.compileToDestination(src, dest);
+ compileProtocol(new File[] {src}, dest);
+ }
+
+ /**
+ * Generates Java interface and classes for a number of protocol files.
+ * @param srcFiles the source Avro protocol files
+ * @param dest the directory to place generated files in
+ */
+ public static void compileProtocol(File[] srcFiles, File dest) throws IOException {
+ for (File src : srcFiles) {
+ Protocol protocol = Protocol.parse(src);
+ SpecificCompiler compiler = new SpecificCompiler(protocol);
+ compiler.compileToDestination(src, dest);
+ }
}
/** Generates Java classes for a schema. */
public static void compileSchema(File src, File dest) throws IOException {
- Schema schema = Schema.parse(src);
- SpecificCompiler compiler = new SpecificCompiler(schema);
- compiler.compileToDestination(src, dest);
+ compileSchema(new File[] {src}, dest);
+ }
+
+ /** Generates Java classes for a number of schema files. */
+ public static void compileSchema(File[] srcFiles, File dest) throws IOException {
+ Schema.Parser parser = new Schema.Parser();
+
+ for (File src : srcFiles) {
+ Schema schema = parser.parse(src);
+ SpecificCompiler compiler = new SpecificCompiler(schema);
+ compiler.compileToDestination(src, dest);
+ }
}
/** Recursively enqueue schemas that need a class generated. */
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java?rev=1161755&r1=1161754&r2=1161755&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/SpecificCompilerTool.java Thu Aug 25 20:51:43 2011
@@ -18,8 +18,12 @@
package org.apache.avro.tool;
import java.io.File;
+import java.io.FilenameFilter;
import java.io.InputStream;
import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.Set;
+import java.util.LinkedHashSet;
import java.util.List;
import org.apache.avro.compiler.specific.SpecificCompiler;
@@ -33,18 +37,27 @@ public class SpecificCompilerTool implem
@Override
public int run(InputStream in, PrintStream out, PrintStream err,
List<String> args) throws Exception {
- if (args.size() != 3) {
+ if (args.size() < 3) {
System.err
- .println("Expected 3 arguments: (schema|protocol) inputfile outputdir");
+ .println("Need at least 3 arguments: (schema|protocol) input... outputdir");
+ System.err
+ .println(" input - input files or directories");
+ System.err
+ .println(" outputdir - directory to write generated java");
return 1;
}
String method = args.get(0);
- File input = new File(args.get(1));
- File output = new File(args.get(2));
+ List<File> inputs = new ArrayList<File>();
+ File output = new File(args.get(args.size() - 1));
+
+ for (int i = 1; i < args.size() - 1; i++) {
+ inputs.add(new File(args.get(i)));
+ }
+
if ("schema".equals(method)) {
- SpecificCompiler.compileSchema(input, output);
+ SpecificCompiler.compileSchema(determineInputs(inputs, SCHEMA_FILTER), output);
} else if ("protocol".equals(method)) {
- SpecificCompiler.compileProtocol(input, output);
+ SpecificCompiler.compileProtocol(determineInputs(inputs, PROTOCOL_FILTER), output);
} else {
System.err.println("Expected \"schema\" or \"protocol\".");
return 1;
@@ -61,4 +74,59 @@ public class SpecificCompilerTool implem
public String getShortDescription() {
return "Generates Java code for the given schema.";
}
+
+ /**
+ * For a List of files or directories, returns a File[] containing each file
+ * passed as well as each file with a matching extension found in the directory.
+ *
+ * @param inputs List of File objects that are files or directories
+ * @param filter File extension filter to match on when fetching files from a directory
+ * @return Unique array of files
+ */
+ private static File[] determineInputs(List<File> inputs, FilenameFilter filter) {
+ Set<File> fileSet = new LinkedHashSet<File>(); // preserve order and uniqueness
+
+ for (File file : inputs) {
+ // if directory, look at contents to see what files match extension
+ if (file.isDirectory()) {
+ for (File f : file.listFiles(filter)) {
+ fileSet.add(f);
+ }
+ }
+ // otherwise, just add the file.
+ else {
+ fileSet.add(file);
+ }
+ }
+
+ if (fileSet.size() > 0) {
+ System.err.println("Input files to compile:");
+ for (File file : fileSet) {
+ System.err.println(" " + file);
+ }
+ }
+ else {
+ System.err.println("No input files found.");
+ }
+
+ return fileSet.toArray((new File[fileSet.size()]));
+ }
+
+ private static final FileExtensionFilter SCHEMA_FILTER =
+ new FileExtensionFilter("avsc");
+ private static final FileExtensionFilter PROTOCOL_FILTER =
+ new FileExtensionFilter("avpr");
+
+ private static class FileExtensionFilter implements FilenameFilter {
+ private String extension;
+
+ private FileExtensionFilter(String extension) {
+ this.extension = extension;
+ }
+
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.endsWith(this.extension);
+ }
+ }
}
Added: avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc?rev=1161755&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc (added)
+++ avro/trunk/lang/java/tools/src/test/compiler/input/player.avsc Thu Aug 25 20:51:43 2011
@@ -0,0 +1,8 @@
+{"type":"record", "name":"Player", "namespace": "avro.examples.baseball",
+ "fields": [
+ {"name": "number", "type": "int"},
+ {"name": "first_name", "type": "string"},
+ {"name": "last_name", "type": "string"},
+ {"name": "position", "type": {"type": "array", "items": "Position"} }
+ ]
+}
Added: avro/trunk/lang/java/tools/src/test/compiler/input/position.avsc
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/input/position.avsc?rev=1161755&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/input/position.avsc (added)
+++ avro/trunk/lang/java/tools/src/test/compiler/input/position.avsc Thu Aug 25 20:51:43 2011
@@ -0,0 +1,3 @@
+{"type":"enum", "name": "Position", "namespace": "avro.examples.baseball",
+ "symbols": ["P", "C", "B1", "B2", "B3", "SS", "LF", "CF", "RF", "DH"]
+}
Added: avro/trunk/lang/java/tools/src/test/compiler/output/Player.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/output/Player.java?rev=1161755&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/output/Player.java (added)
+++ avro/trunk/lang/java/tools/src/test/compiler/output/Player.java Thu Aug 25 20:51:43 2011
@@ -0,0 +1,36 @@
+/**
+ * Autogenerated by Avro
+ *
+ * DO NOT EDIT DIRECTLY
+ */
+package avro.examples.baseball;
+@SuppressWarnings("all")
+public class Player extends org.apache.avro.specific.SpecificRecordBase implements org.apache.avro.specific.SpecificRecord {
+ public static final org.apache.avro.Schema SCHEMA$ = org.apache.avro.Schema.parse("{\"type\":\"record\",\"name\":\"Player\",\"namespace\":\"avro.examples.baseball\",\"fields\":[{\"name\":\"number\",\"type\":\"int\"},{\"name\":\"first_name\",\"type\":\"string\"},{\"name\":\"last_name\",\"type\":\"string\"},{\"name\":\"position\",\"type\":{\"type\":\"array\",\"items\":{\"type\":\"enum\",\"name\":\"Position\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}}}]}");
+ public int number;
+ public java.lang.CharSequence first_name;
+ public java.lang.CharSequence last_name;
+ public java.util.List<avro.examples.baseball.Position> position;
+ public org.apache.avro.Schema getSchema() { return SCHEMA$; }
+ // Used by DatumWriter. Applications should not call.
+ public java.lang.Object get(int field$) {
+ switch (field$) {
+ case 0: return number;
+ case 1: return first_name;
+ case 2: return last_name;
+ case 3: return position;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+ }
+ }
+ // Used by DatumReader. Applications should not call.
+ @SuppressWarnings(value="unchecked")
+ public void put(int field$, java.lang.Object value$) {
+ switch (field$) {
+ case 0: number = (java.lang.Integer)value$; break;
+ case 1: first_name = (java.lang.CharSequence)value$; break;
+ case 2: last_name = (java.lang.CharSequence)value$; break;
+ case 3: position = (java.util.List<avro.examples.baseball.Position>)value$; break;
+ default: throw new org.apache.avro.AvroRuntimeException("Bad index");
+ }
+ }
+}
Added: avro/trunk/lang/java/tools/src/test/compiler/output/Position.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/compiler/output/Position.java?rev=1161755&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/test/compiler/output/Position.java (added)
+++ avro/trunk/lang/java/tools/src/test/compiler/output/Position.java Thu Aug 25 20:51:43 2011
@@ -0,0 +1,11 @@
+/**
+ * Autogenerated by Avro
+ *
+ * DO NOT EDIT DIRECTLY
+ */
+package avro.examples.baseball;
+@SuppressWarnings("all")
+public enum Position {
+ P, C, B1, B2, B3, SS, LF, CF, RF, DH ;
+ public static final org.apache.avro.Schema SCHEMA$ = org.apache.avro.Schema.parse("{\"type\":\"enum\",\"name\":\"Position\",\"namespace\":\"avro.examples.baseball\",\"symbols\":[\"P\",\"C\",\"B1\",\"B2\",\"B3\",\"SS\",\"LF\",\"CF\",\"RF\",\"DH\"]}");
+}
Added: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java?rev=1161755&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java (added)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestSpecificCompilerTool.java Thu Aug 25 20:51:43 2011
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.avro.tool;
+
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Verifies that the SpecificCompilerTool generates Java source properly
+ */
+public class TestSpecificCompilerTool {
+
+ // where test input/expected output comes from
+ private static final File TEST_DIR =
+ new File(System.getProperty("test.compile.schema.dir", "src/test/compiler"));
+
+ // where test input comes from
+ private static final File TEST_INPUT_DIR =
+ new File(TEST_DIR, "input");
+
+ // where test expected output comes from
+ private static final File TEST_EXPECTED_OUTPUT_DIR =
+ new File(TEST_DIR, "output");
+ private static final File TEST_EXPECTED_POSITION =
+ new File(TEST_EXPECTED_OUTPUT_DIR, "Position.java");
+ private static final File TEST_EXPECTED_PLAYER =
+ new File(TEST_EXPECTED_OUTPUT_DIR, "Player.java");
+
+ // where test output goes
+ private static final File TEST_OUTPUT_DIR =
+ new File("target/compiler/output");
+ private static final File TEST_OUTPUT_PLAYER =
+ new File(TEST_OUTPUT_DIR, "avro/examples/baseball/Player.java");
+ private static final File TEST_OUTPUT_POSITION =
+ new File(TEST_OUTPUT_DIR, "avro/examples/baseball/Position.java");
+
+ @Before
+ public void setUp() {
+ TEST_OUTPUT_DIR.delete();
+ }
+
+ @Test
+ public void testCompileSchemaSingleFile() throws Exception {
+
+ doCompile(new String[]{"schema",
+ TEST_INPUT_DIR.toString() + "/position.avsc",
+ TEST_OUTPUT_DIR.getPath()});
+ assertFileMatch(TEST_EXPECTED_POSITION, TEST_OUTPUT_POSITION);
+ }
+
+ @Test
+ public void testCompileSchemaTwoFiles() throws Exception {
+
+ doCompile(new String[]{"schema",
+ TEST_INPUT_DIR.toString() + "/position.avsc",
+ TEST_INPUT_DIR.toString() + "/player.avsc",
+ TEST_OUTPUT_DIR.getPath()});
+ assertFileMatch(TEST_EXPECTED_POSITION, TEST_OUTPUT_POSITION);
+ assertFileMatch(TEST_EXPECTED_PLAYER, TEST_OUTPUT_PLAYER);
+ }
+
+ @Test
+ public void testCompileSchemaFileAndDirectory() throws Exception {
+
+ doCompile(new String[]{"schema",
+ TEST_INPUT_DIR.toString() + "/position.avsc",
+ TEST_INPUT_DIR.toString(),
+ TEST_OUTPUT_DIR.getPath()});
+ assertFileMatch(TEST_EXPECTED_POSITION, TEST_OUTPUT_POSITION);
+ assertFileMatch(TEST_EXPECTED_PLAYER, TEST_OUTPUT_PLAYER);
+ }
+
+ // Runs the actual compiler tool with the given input args
+ private void doCompile(String[] args) throws Exception {
+ SpecificCompilerTool tool = new SpecificCompilerTool();
+ tool.run(null, null, null, Arrays.asList((args)));
+ }
+
+ /**
+ * Verify that the generated Java files match the expected. This approach has
+ * room for improvement, since we're currently just verify that the text matches,
+ * which can be brittle if the code generation formatting or method ordering
+ * changes for example. A better approach would be to compile the sources and
+ * do a deeper comparison.
+ *
+ * See http://download.oracle.com/javase/6/docs/api/javax/tools/JavaCompiler.html
+ */
+ private static void assertFileMatch(File expected, File found) throws IOException {
+ Assert.assertEquals("Found file: " + found +
+ " does not match expected file: " + expected,
+ readFile(expected), readFile(found));
+ }
+
+ /**
+ * Not the best implementation, but does the job. Building full strings of the
+ * file content and comparing provides nice diffs via JUnit when failures occur.
+ */
+ private static String readFile(File file) throws IOException {
+ BufferedReader reader = new BufferedReader(new FileReader(file));
+ StringBuilder sb = new StringBuilder();
+ String line = null;
+ boolean first = true;
+ while ((line = reader.readLine()) != null) {
+ if (!first) {
+ sb.append("\n");
+ first = false;
+ }
+ sb.append(line);
+ }
+ return sb.toString();
+ }
+}