You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by rd...@apache.org on 2010/08/31 00:57:33 UTC
svn commit: r991018 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/Main.java
src/org/apache/pig/tools/pigstats/ScriptState.java
test/org/apache/pig/test/TestPigRunner.java
Author: rding
Date: Mon Aug 30 22:57:33 2010
New Revision: 991018
URL: http://svn.apache.org/viewvc?rev=991018&view=rev
Log:
PIG-1569: java properties not honored in case of properties such as stop.on.failure
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/Main.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=991018&r1=991017&r2=991018&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Aug 30 22:57:33 2010
@@ -175,6 +175,9 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1569: java properties not honored in case of properties such as
+stop.on.failure (rding)
+
PIG-1570: native mapreduce operator MR job does not follow same failure handling logic as other pig MR jobs (thejas)
PIG-1343: pig_log file missing even though Main tells it is creating one and
Modified: hadoop/pig/trunk/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/Main.java?rev=991018&r1=991017&r2=991018&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/Main.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/Main.java Mon Aug 30 22:57:33 2010
@@ -111,10 +111,11 @@ static int run(String args[], PigProgres
int rc = 1;
GenericOptionsParser parser = new GenericOptionsParser(args);
- Configuration conf = parser.getConfiguration();
- Properties properties = ConfigurationUtil.toProperties(conf);
+ Configuration conf = parser.getConfiguration();
+ Properties properties = new Properties();
PropertiesUtil.loadDefaultProperties(properties);
+ properties.putAll(ConfigurationUtil.toProperties(conf));
String[] pigArgs = parser.getRemainingArgs();
@@ -161,15 +162,21 @@ static int run(String args[], PigProgres
if(execTypeString!=null && execTypeString.length()>0){
execType = PigServer.parseExecType(execTypeString);
}
-
- //by default warning aggregation is on
- properties.setProperty("aggregate.warning", ""+true);
-
- //by default multiquery optimization is on
- properties.setProperty("opt.multiquery", ""+true);
+
+ if (properties.getProperty("aggregate.warning") == null) {
+ //by default warning aggregation is on
+ properties.setProperty("aggregate.warning", ""+true);
+ }
+
+ if (properties.getProperty("opt.multiquery") == null) {
+ //by default multiquery optimization is on
+ properties.setProperty("opt.multiquery", ""+true);
+ }
- //by default we keep going on error on the backend
- properties.setProperty("stop.on.failure", ""+false);
+ if (properties.getProperty("stop.on.failure") == null) {
+ //by default we keep going on error on the backend
+ properties.setProperty("stop.on.failure", ""+false);
+ }
// set up client side system properties in UDF context
UDFContext.getUDFContext().setClientSystemProps();
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=991018&r1=991017&r2=991018&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Mon Aug 30 22:57:33 2010
@@ -353,7 +353,7 @@ public class ScriptState {
Attributes attr = attrs.get("org/apache/pig");
pigVersion = attr.getValue("Implementation-Version");
} catch (Exception e) {
- LOG.warn("unable to read pigs manifest file", e);
+ LOG.warn("unable to read pigs manifest file");
}
}
return (pigVersion == null) ? "" : pigVersion;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=991018&r1=991017&r2=991018&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java Mon Aug 30 22:57:33 2010
@@ -30,10 +30,12 @@ import java.util.Properties;
import junit.framework.Assert;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.PigRunner.ReturnCode;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.newplan.Operator;
@@ -72,7 +74,7 @@ public class TestPigRunner {
public static void tearDownAfterClass() throws Exception {
new File(INPUT_FILE).delete();
cluster.shutDown();
- }
+ }
@Before
public void setUp() {
@@ -135,12 +137,12 @@ public class TestPigRunner {
w.close();
try {
- String[] args = { PIG_FILE };
+ String[] args = { "-Dstop.on.failure=true", "-Dopt.multiquery=false", "-Daggregate.warning=false", PIG_FILE };
PigStats stats = PigRunner.run(args, new TestNotificationListener());
assertTrue(stats.isSuccessful());
- assertTrue(stats.getJobGraph().size() == 1);
+ assertEquals(1, stats.getNumberJobs());
String name = stats.getOutputNames().get(0);
assertEquals(OUTPUT_FILE, name);
assertEquals(12, stats.getBytesWritten());
@@ -148,6 +150,11 @@ public class TestPigRunner {
assertEquals("A,B,C",
((JobStats)stats.getJobGraph().getSinks().get(0)).getAlias());
+
+ Configuration conf = ConfigurationUtil.toConfiguration(stats.getPigProperties());
+ assertTrue(conf.getBoolean("stop.on.failure", false));
+ assertTrue(!conf.getBoolean("aggregate.warning", true));
+ assertTrue(!conf.getBoolean("opt.multiquery", true));
} finally {
new File(PIG_FILE).delete();
Util.deleteFile(cluster, OUTPUT_FILE);