You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by rk...@apache.org on 2013/09/23 20:24:23 UTC
svn commit: r1525669 - in /oozie/trunk:
core/src/test/java/org/apache/oozie/action/hadoop/MapperReducerUberJarForTest.java
release-log.txt
sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
Author: rkanter
Date: Mon Sep 23 18:24:23 2013
New Revision: 1525669
URL: http://svn.apache.org/r1525669
Log:
OOZIE-1546 TestMapReduceActionExecutorUberJar.testMapReduceWithUberJarEnabled fails (rkanter)
Modified:
oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/MapperReducerUberJarForTest.java
oozie/trunk/release-log.txt
oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
Modified: oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/MapperReducerUberJarForTest.java
URL: http://svn.apache.org/viewvc/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/MapperReducerUberJarForTest.java?rev=1525669&r1=1525668&r2=1525669&view=diff
==============================================================================
--- oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/MapperReducerUberJarForTest.java (original)
+++ oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/MapperReducerUberJarForTest.java Mon Sep 23 18:24:23 2013
@@ -27,10 +27,11 @@ import java.io.IOException;
import java.net.URL;
import java.net.URLClassLoader;
import java.util.Iterator;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
+/**
+ * This is just like MapperReducerForTest except that this map function outputs the classpath as the value
+ */
public class MapperReducerUberJarForTest implements Mapper, Reducer {
public static final String GROUP = "g";
public static final String NAME = "c";
@@ -45,18 +46,18 @@ public class MapperReducerUberJarForTest
public void close() throws IOException {
}
- private static final LongWritable zero = new LongWritable(0);
-
@SuppressWarnings("unchecked")
public void map(Object key, Object value, OutputCollector collector, Reporter reporter) throws IOException {
+ StringBuilder sb = new StringBuilder();
ClassLoader applicationClassLoader = this.getClass().getClassLoader();
if (applicationClassLoader == null) {
applicationClassLoader = ClassLoader.getSystemClassLoader();
}
URL[] urls = ((URLClassLoader) applicationClassLoader).getURLs();
for (URL url : urls) {
- collector.collect(zero, new Text(url.toString()));
+ sb.append(url.toString()).append("@");
}
+ collector.collect(key, new Text(sb.toString()));
reporter.incrCounter(GROUP, NAME, 5l);
}
@@ -64,7 +65,7 @@ public class MapperReducerUberJarForTest
public void reduce(Object key, Iterator values, OutputCollector collector, Reporter reporter)
throws IOException {
while (values.hasNext()) {
- collector.collect(values.next(), NullWritable.get());
+ collector.collect(key, values.next());
}
}
}
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1525669&r1=1525668&r2=1525669&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Mon Sep 23 18:24:23 2013
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1546 TestMapReduceActionExecutorUberJar.testMapReduceWithUberJarEnabled fails (rkanter)
OOZIE-1545 RecoveryService keeps repeatedly queueing SuspendXCommand (rohini)
OOZIE-1547 Change Coordinator SELECT query to fetch only necessary columns and consolidate JPA Executors (ryota)
OOZIE-1529 Disable job DAG display for workflow having huge actions (puru via rohini)
Modified: oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java
URL: http://svn.apache.org/viewvc/oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java?rev=1525669&r1=1525668&r2=1525669&view=diff
==============================================================================
--- oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java (original)
+++ oozie/trunk/sharelib/streaming/src/test/java/org/apache/oozie/action/hadoop/TestMapReduceActionExecutor.java Mon Sep 23 18:24:23 2013
@@ -502,11 +502,11 @@ public class TestMapReduceActionExecutor
boolean containsLib1Jar = false;
String lib1JarStr = "jobcache/" + jobID + "/jars/lib/lib1.jar";
Pattern lib1JarPatYarn = Pattern.compile(
- ".*appcache/application_" + jobID.replaceFirst("job_", "") + "/filecache/.*/uber.jar/lib/lib1.jar");
+ ".*appcache/application_" + jobID.replaceFirst("job_", "") + "/filecache/.*/uber.jar/lib/lib1.jar.*");
boolean containsLib2Jar = false;
String lib2JarStr = "jobcache/" + jobID + "/jars/lib/lib1.jar";
Pattern lib2JarPatYarn = Pattern.compile(
- ".*appcache/application_" + jobID.replaceFirst("job_", "") + "/filecache/.*/uber.jar/lib/lib2.jar");
+ ".*appcache/application_" + jobID.replaceFirst("job_", "") + "/filecache/.*/uber.jar/lib/lib2.jar.*");
FileStatus[] fstats = getFileSystem().listStatus(outputDir);
for (FileStatus fstat : fstats) {
@@ -516,8 +516,8 @@ public class TestMapReduceActionExecutor
Scanner sc = new Scanner(is);
while (sc.hasNextLine()) {
String line = sc.nextLine();
- containsLib1Jar = (containsLib1Jar || line.endsWith(lib1JarStr) || lib1JarPatYarn.matcher(line).matches());
- containsLib2Jar = (containsLib2Jar || line.endsWith(lib2JarStr) || lib2JarPatYarn.matcher(line).matches());
+ containsLib1Jar = (containsLib1Jar || line.contains(lib1JarStr) || lib1JarPatYarn.matcher(line).matches());
+ containsLib2Jar = (containsLib2Jar || line.contains(lib2JarStr) || lib2JarPatYarn.matcher(line).matches());
}
sc.close();
is.close();