You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/03/16 23:19:41 UTC
svn commit: r1301795 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/impl/util/ test/e2e/pig/tests/
test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/
Author: daijy
Date: Fri Mar 16 22:19:41 2012
New Revision: 1301795
URL: http://svn.apache.org/viewvc?rev=1301795&view=rev
Log:
PIG-2576: Change in behavior for UDFContext.getUDFContext().getJobConf() in front-end
Added:
pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UdfContextFrontend.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1301795&r1=1301794&r2=1301795&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Mar 16 22:19:41 2012
@@ -437,6 +437,8 @@ Release 0.9.3 - Unreleased
BUG FIXES
+PIG-2576: Change in behavior for UDFContext.getUDFContext().getJobConf() in front-end (thw via daijy)
+
PIG-2588: e2e harness: use pig command for cluster deploy (thw via daijy)
PIG-2572: e2e harness deploy fails when using pig that does not bundle hadoop (thw via daijy)
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1301795&r1=1301794&r2=1301795&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java Fri Mar 16 22:19:41 2012
@@ -256,7 +256,7 @@ public class MapReduceLauncher extends L
Thread jcThread = new Thread(jc) {
@Override
public void run() {
- UDFContext.setUdfContext(udfContext);
+ UDFContext.setUdfContext(udfContext.clone()); //PIG-2576
super.run();
}
};
Modified: pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/UDFContext.java?rev=1301795&r1=1301794&r2=1301795&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/UDFContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/UDFContext.java Fri Mar 16 22:19:41 2012
@@ -205,6 +205,25 @@ public class UDFContext {
public boolean isUDFConfEmpty() {
return udfConfs.isEmpty();
}
+
+ /**
+ * Convenience method for UDF code to check where it runs (see PIG-2576)
+ * @return
+ */
+ public boolean isFrontend() {
+ return (this.jconf == null || jconf.get("mapred.task.id") == null);
+ }
+
+ /**
+ * Make a shallow copy of the context.
+ */
+ public UDFContext clone() {
+ UDFContext other = new UDFContext();
+ other.clientSysProps = this.clientSysProps;
+ other.jconf = this.jconf;
+ other.udfConfs = this.udfConfs;
+ return other;
+ }
/**
* Class that acts as key for hashmap in UDFContext,
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1301795&r1=1301794&r2=1301795&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Fri Mar 16 22:19:41 2012
@@ -4043,6 +4043,21 @@ store E into ':OUTPATH:';\,
STORE B1 INTO ':OUTPATH:.1';
C = DISTINCT B1 ;
store C into ':OUTPATH:.2';?,
+ }, {
+ # PIG-2576
+ 'num' => 4,
+ 'execonly' => 'mapred',
+ 'pig' => q?register :FUNCPATH:/testudf.jar;
+ define printconf org.apache.pig.test.udf.evalfunc.UdfContextFrontend('dummy');
+ a = load ':INPATH:/singlefile/studenttab10k' as (name, age, gpa);
+ b = limit a 1;
+ c = foreach b generate printconf(name);
+ store c into ':OUTPATH:';
+ fs -ls;
+ ?,
+ 'rc' => 0,
+ 'not_expected_out_regex' => "checkJobConf: conf is null: false",
+ 'expected_out_regex' => "checkJobConf: conf is null: true",
}
],
},{
Added: pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UdfContextFrontend.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UdfContextFrontend.java?rev=1301795&view=auto
==============================================================================
--- pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UdfContextFrontend.java (added)
+++ pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/UdfContextFrontend.java Fri Mar 16 22:19:41 2012
@@ -0,0 +1,27 @@
+package org.apache.pig.test.udf.evalfunc;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.util.UDFContext;
+
+public class UdfContextFrontend extends EvalFunc<Boolean> {
+
+ public UdfContextFrontend(){}
+
+ public UdfContextFrontend(String param){
+ checkJobConf(); // call here and it will execute multiple times in front end
+ }
+
+ private void checkJobConf() {
+ Configuration jobConf = UDFContext.getUDFContext().getJobConf();
+ System.out.println("checkJobConf: conf is null: " + (jobConf == null) + " conf: " + jobConf);
+ }
+
+ public Boolean exec(Tuple input) throws IOException {
+ checkJobConf(); // will execute in map only
+ return true;
+ }
+}