You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by sc...@apache.org on 2011/01/14 20:41:04 UTC
svn commit: r1059139 - in /avro/trunk: ./
lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/
lang/java/tools/ lang/java/tools/src/main/java/org/apache/avro/tool/
lang/java/tools/src/test/java/org/apache/avro/tool/
Author: scottcarey
Date: Fri Jan 14 19:41:03 2011
New Revision: 1059139
URL: http://svn.apache.org/viewvc?rev=1059139&view=rev
Log:
AVRO-663 Java build: Change build to create an avro-tools.jar that conforms to maven standards and contains dependencies within
Added:
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
avro/trunk/lang/java/tools/pom.xml
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Jan 14 19:41:03 2011
@@ -26,7 +26,8 @@ Avro 1.5.0 (unreleased)
API for Hadoop MapReduce with Avro data.
Dependencies: avro-ipc.jar, hadoop-core, and jopt-simple.
* avro-tools.jar
- Avro command-line tools.
+ Avro command-line tools stand-alone jar.
+ Dependencies are contained within the jar.
Dependencies: all of the above.
(scottcarey)
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java Fri Jan 14 19:41:03 2011
@@ -19,10 +19,6 @@
package org.apache.avro.mapred.tether;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.io.File;
-import java.util.List;
import java.util.Collection;
import java.net.URI;
import java.net.URISyntaxException;
@@ -30,24 +26,14 @@ import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.filecache.DistributedCache;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapred.FileInputFormat;
-import org.apache.hadoop.mapred.FileOutputFormat;
import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RunningJob;
-import joptsimple.OptionParser;
-import joptsimple.OptionSet;
-import joptsimple.OptionSpec;
-
-import org.apache.avro.Schema;
-import org.apache.avro.tool.Tool;
-import org.apache.avro.mapred.AvroJob;
-
-/** Constructs and submits tether jobs. This may either be used as a
- * commandline-based or API-based method to launch tether jobs. */
-public class TetherJob extends Configured implements Tool {
+/** Constructs and submits tether jobs. This may be used as an API-based
+ * method to launch tether jobs. */
+@SuppressWarnings("deprecation")
+public class TetherJob extends Configured {
/** Get the URI of the application's executable. */
public static URI getExecutable(JobConf job) {
@@ -102,55 +88,4 @@ public class TetherJob extends Configure
DistributedCache.addCacheFile(getExecutable(job), job);
}
- // Tool methods
-
- @Override
- public String getName() { return "tether"; }
-
- @Override
- public String getShortDescription() {return "Run a tethered mapreduce job.";}
-
- @Override
- public int run(InputStream ins, PrintStream outs, PrintStream err,
- List<String> args) throws Exception {
-
- OptionParser p = new OptionParser();
- OptionSpec<URI> exec =
- p.accepts("program", "executable program, usually in HDFS")
- .withRequiredArg().ofType(URI.class);
- OptionSpec<String> in = p.accepts("in", "comma-separated input paths")
- .withRequiredArg().ofType(String.class);
- OptionSpec<Path> out = p.accepts("out", "output directory")
- .withRequiredArg().ofType(Path.class);
- OptionSpec<File> outSchema = p.accepts("outschema", "output schema file")
- .withRequiredArg().ofType(File.class);
- OptionSpec<File> mapOutSchema =
- p.accepts("outschemamap", "map output schema file, if different")
- .withOptionalArg().ofType(File.class);
- OptionSpec<Integer> reduces = p.accepts("reduces", "number of reduces")
- .withOptionalArg().ofType(Integer.class);
-
- JobConf job = new JobConf();
-
- try {
- OptionSet opts = p.parse(args.toArray(new String[0]));
- FileInputFormat.addInputPaths(job, in.value(opts));
- FileOutputFormat.setOutputPath(job, out.value(opts));
- TetherJob.setExecutable(job, exec.value(opts));
- job.set(AvroJob.OUTPUT_SCHEMA,
- Schema.parse(outSchema.value(opts)).toString());
- if (opts.hasArgument(mapOutSchema))
- job.set(AvroJob.MAP_OUTPUT_SCHEMA,
- Schema.parse(mapOutSchema.value(opts)).toString());
- if (opts.hasArgument(reduces))
- job.setNumReduceTasks(reduces.value(opts));
- } catch (Exception e) {
- p.printHelpOn(err);
- return -1;
- }
-
- runJob(job);
- return 0;
- }
-
}
Modified: avro/trunk/lang/java/tools/pom.xml
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/pom.xml?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/pom.xml (original)
+++ avro/trunk/lang/java/tools/pom.xml Fri Jan 14 19:41:03 2011
@@ -31,17 +31,40 @@
<description>Avro command line tools and utilities</description>
<build>
+ <resources>
+ <resource>
+ <includes>
+ <include>VERSION.txt</include>
+ </includes>
+ <directory>../../../share/</directory>
+ </resource>
+ <resource>
+ <includes>
+ <include>NOTICE.txt</include>
+ </includes>
+ <directory>../../../</directory>
+ </resource>
+ </resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifest>
- <mainClass>org.apache.avro.tool.Main</mainClass>
- </manifest>
- </archive>
- </configuration>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>1.4</version>
+ <executions>
+ <execution>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>org.apache.avro.tool.Main</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
</plugin>
</plugins>
</build>
@@ -70,6 +93,81 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-api-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jsp-2.1</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>net.java.dev.jets3t</groupId>
+ <artifactId>jets3t</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-net</groupId>
+ <artifactId>commons-net</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-el</groupId>
+ <artifactId>commons-el</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>xmlenc</groupId>
+ <artifactId>xmlenc</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.eclipse.jdt</groupId>
+ <artifactId>core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>hsqldb</groupId>
+ <artifactId>hsqldb</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>oro</groupId>
+ <artifactId>oro</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>net.sf.jopt-simple</groupId>
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/Main.java Fri Jan 14 19:41:03 2011
@@ -23,7 +23,6 @@ import java.util.TreeMap;
import java.io.InputStream;
-import org.apache.avro.mapred.tether.TetherJob;
import org.apache.avro.reflect.InduceSchemaTool;
import org.apache.avro.specific.SpecificCompiler.SpecificCompilerTool;
@@ -52,7 +51,7 @@ public class Main {
new RpcSendTool(),
new FromTextTool(),
new ToTextTool(),
- new TetherJob()
+ new TetherTool()
}) {
Tool prev = tools.put(tool.getName(), tool);
if (prev != null) {
Added: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java?rev=1059139&view=auto
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java (added)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java Fri Jan 14 19:41:03 2011
@@ -0,0 +1,78 @@
+package org.apache.avro.tool;
+
+import java.io.File;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.net.URI;
+import java.util.List;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+
+import org.apache.avro.Schema;
+import org.apache.avro.mapred.AvroJob;
+import org.apache.avro.mapred.tether.TetherJob;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobConf;
+
+@SuppressWarnings("deprecation")
+public class TetherTool implements Tool {
+ public TetherJob job;
+
+ @Override
+ public String getName() {
+ return "tether";
+ }
+
+ @Override
+ public String getShortDescription() {
+ return "Run a tethered mapreduce job.";
+ }
+
+ @Override
+ public int run(InputStream ins, PrintStream outs, PrintStream err,
+ List<String> args) throws Exception {
+
+ OptionParser p = new OptionParser();
+ OptionSpec<URI> exec = p
+ .accepts("program", "executable program, usually in HDFS")
+ .withRequiredArg().ofType(URI.class);
+ OptionSpec<String> in = p.accepts("in", "comma-separated input paths")
+ .withRequiredArg().ofType(String.class);
+ OptionSpec<Path> out = p.accepts("out", "output directory")
+ .withRequiredArg().ofType(Path.class);
+ OptionSpec<File> outSchema = p.accepts("outschema", "output schema file")
+ .withRequiredArg().ofType(File.class);
+ OptionSpec<File> mapOutSchema = p
+ .accepts("outschemamap", "map output schema file, if different")
+ .withOptionalArg().ofType(File.class);
+ OptionSpec<Integer> reduces = p.accepts("reduces", "number of reduces")
+ .withOptionalArg().ofType(Integer.class);
+
+ JobConf job = new JobConf();
+
+ try {
+ OptionSet opts = p.parse(args.toArray(new String[0]));
+ FileInputFormat.addInputPaths(job, in.value(opts));
+ FileOutputFormat.setOutputPath(job, out.value(opts));
+ TetherJob.setExecutable(job, exec.value(opts));
+ job.set(AvroJob.OUTPUT_SCHEMA, Schema.parse(outSchema.value(opts))
+ .toString());
+ if (opts.hasArgument(mapOutSchema))
+ job.set(AvroJob.MAP_OUTPUT_SCHEMA,
+ Schema.parse(mapOutSchema.value(opts)).toString());
+ if (opts.hasArgument(reduces))
+ job.setNumReduceTasks(reduces.value(opts));
+ } catch (Exception e) {
+ p.printHelpOn(err);
+ return -1;
+ }
+
+ TetherJob.runJob(job);
+ return 0;
+ }
+
+}
Modified: avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java?rev=1059139&r1=1059138&r2=1059139&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java (original)
+++ avro/trunk/lang/java/tools/src/test/java/org/apache/avro/tool/TestDataFileTools.java Fri Jan 14 19:41:03 2011
@@ -23,9 +23,10 @@ import static org.junit.Assert.fail;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
-import java.io.FileWriter;
+import java.io.StringBufferInputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -38,10 +39,10 @@ import org.apache.avro.file.DataFileRead
import org.apache.avro.file.DataFileWriter;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.generic.GenericDatumWriter;
-import org.apache.tools.ant.filters.StringInputStream;
import org.junit.BeforeClass;
import org.junit.Test;
+@SuppressWarnings("deprecation")
public class TestDataFileTools {
static final int COUNT = 10;
static File sampleFile;
@@ -130,7 +131,7 @@ public class TestDataFileTools {
args.add("-");
args.addAll(extra);
new DataFileWriteTool().run(
- new StringInputStream(jsonData),
+ new StringBufferInputStream(jsonData),
new PrintStream(out), // stdout
null, // stderr
args);
@@ -160,7 +161,7 @@ public class TestDataFileTools {
PrintStream out = new PrintStream(baos);
try {
new DataFileWriteTool().run(
- new StringInputStream("{"),
+ new StringBufferInputStream("{"),
new PrintStream(out), // stdout
null, // stderr
Arrays.asList("-schema", "{ \"type\":\"record\", \"fields\":" +
@@ -206,7 +207,7 @@ public class TestDataFileTools {
FileOutputStream fout = new FileOutputStream(outFile);
PrintStream out = new PrintStream(fout);
new DataFileWriteTool().run(
- new StringInputStream(json),
+ new StringBufferInputStream(json),
new PrintStream(out), // stdout
null, // stderr
Arrays.asList("-schema", schema, "-"));