You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@avro.apache.org by th...@apache.org on 2011/10/07 09:20:50 UTC
svn commit: r1179946 - in /avro/trunk: ./
lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/
lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/
lang/java/tools/src/main/java/org/apache/avro/tool/
Author: thiru
Date: Fri Oct 7 07:20:49 2011
New Revision: 1179946
URL: http://svn.apache.org/viewvc?rev=1179946&view=rev
Log:
AVRO-912. Mapreduce tether test fails on Windows
Modified:
avro/trunk/CHANGES.txt
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java
avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java
avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java
Modified: avro/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/avro/trunk/CHANGES.txt?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/CHANGES.txt (original)
+++ avro/trunk/CHANGES.txt Fri Oct 7 07:20:49 2011
@@ -119,6 +119,8 @@ Avro 1.6.0 (unreleased)
AVRO-900. On slower machines Java unit test TestFileSpanStorage fails. (thiru)
+ AVRO-912. Mapreduce tether test fails on Windows. (thiru)
+
Avro 1.5.4 (12 September 2011)
IMPROVEMENTS
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetherJob.java Fri Oct 7 07:20:49 2011
@@ -18,8 +18,11 @@
package org.apache.avro.mapred.tether;
+import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.List;
import java.net.URI;
import java.net.URISyntaxException;
@@ -49,8 +52,8 @@ public class TetherJob extends Configure
}
/** Set the URI for the application's executable. Normally this in HDFS. */
- public static void setExecutable(JobConf job, URI executable) {
- setExecutable(job,executable,"",false);
+ public static void setExecutable(JobConf job, File executable) {
+ setExecutable(job,executable, new ArrayList<String>(),false);
}
/**
@@ -63,9 +66,14 @@ public class TetherJob extends Configure
* - if false its not cached. I.e if the file is already stored on each local file system
* or if its on a NFS share
*/
- public static void setExecutable(JobConf job, URI executable,String argstr,boolean cached) {
+ public static void setExecutable(JobConf job, File executable, List<String> args, boolean cached) {
job.set(TETHER_EXEC, executable.toString());
- job.set(TETHER_EXEC_ARGS, argstr);
+ StringBuilder sb = new StringBuilder();
+ for (String a : args) {
+ sb.append(a);
+ sb.append('\n');
+ }
+ job.set(TETHER_EXEC_ARGS, sb.toString());
job.set(TETHER_EXEC_CACHED, (new Boolean(cached)).toString());
}
Modified: avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java (original)
+++ avro/trunk/lang/java/mapred/src/main/java/org/apache/avro/mapred/tether/TetheredProcess.java Fri Oct 7 07:20:49 2011
@@ -137,7 +137,7 @@ class TetheredProcess {
// if we pass a single string containing all arguments we get quoted
// incorrectly
String args=job.get(TetherJob.TETHER_EXEC_ARGS);
- String[] aparams=args.split(" ");
+ String[] aparams=args.split("\n");
for (int i=0;i<aparams.length; i++){
aparams[i]=aparams[i].trim();
if (aparams[i].length()>0){
@@ -165,6 +165,7 @@ class TetheredProcess {
// start child process
ProcessBuilder builder = new ProcessBuilder(command);
+ System.out.println(command);
builder.environment().putAll(env);
return builder.start();
}
Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java (original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/TestWordCountTether.java Fri Oct 7 07:20:49 2011
@@ -72,7 +72,7 @@ import org.apache.avro.file.DataFileStre
public class TestWordCountTether {
- @Test
+ @Test
@SuppressWarnings("deprecation")
public void testJob() throws Exception {
@@ -86,19 +86,21 @@ public class TestWordCountTether {
// create the input file
WordCountUtil.writeLinesFile();
- java.net.URI exec =
- new java.net.URI(System.getProperty("java.home")+"/bin/java");
+ File exec =
+ new File(System.getProperty("java.home")+"/bin/java");
//input path
String in=dir+"/in";
//create a string of the arguments
- String execargs="-classpath " + System.getProperty("java.class.path");
- execargs+= " org.apache.avro.mapred.tether.WordCountTask";
+ List<String> execargs = new ArrayList<String>();
+ execargs.add("-classpath");
+ execargs.add(System.getProperty("java.class.path"));
+ execargs.add("org.apache.avro.mapred.tether.WordCountTask");
FileInputFormat.addInputPaths(job, in);
FileOutputFormat.setOutputPath(job, outputPath);
- TetherJob.setExecutable(job, exec,execargs,false);
+ TetherJob.setExecutable(job, exec, execargs, false);
Schema outscheme= new Pair<Utf8,Long>(new Utf8(""), 0L).getSchema();
AvroJob.setInputSchema(job, Schema.create(Schema.Type.STRING));
Modified: avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java (original)
+++ avro/trunk/lang/java/mapred/src/test/java/org/apache/avro/mapred/tether/WordCountTask.java Fri Oct 7 07:20:49 2011
@@ -21,8 +21,8 @@ package org.apache.avro.mapred.tether;
import java.io.IOException;
import java.util.StringTokenizer;
-import org.apache.avro.util.Utf8;
import org.apache.avro.mapred.Pair;
+import org.apache.avro.util.Utf8;
/** Example Java tethered mapreduce executable. Implements map and reduce
* functions for word count. */
Modified: avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java
URL: http://svn.apache.org/viewvc/avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java?rev=1179946&r1=1179945&r2=1179946&view=diff
==============================================================================
--- avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java (original)
+++ avro/trunk/lang/java/tools/src/main/java/org/apache/avro/tool/TetherTool.java Fri Oct 7 07:20:49 2011
@@ -20,7 +20,6 @@ package org.apache.avro.tool;
import java.io.File;
import java.io.InputStream;
import java.io.PrintStream;
-import java.net.URI;
import java.util.List;
import joptsimple.OptionParser;
@@ -54,9 +53,9 @@ public class TetherTool implements Tool
List<String> args) throws Exception {
OptionParser p = new OptionParser();
- OptionSpec<URI> exec = p
+ OptionSpec<File> exec = p
.accepts("program", "executable program, usually in HDFS")
- .withRequiredArg().ofType(URI.class);
+ .withRequiredArg().ofType(File.class);
OptionSpec<String> in = p.accepts("in", "comma-separated input paths")
.withRequiredArg().ofType(String.class);
OptionSpec<Path> out = p.accepts("out", "output directory")