You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2014/05/30 02:28:29 UTC
svn commit: r1598445 - in /pig/trunk: ./ src/org/apache/pig/
src/org/apache/pig/newplan/logical/relational/ test/org/apache/pig/test/
Author: daijy
Date: Fri May 30 00:28:29 2014
New Revision: 1598445
URL: http://svn.apache.org/r1598445
Log:
PIG-3967: Grunt fail if we running more statement after first store
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java
pig/trunk/test/org/apache/pig/test/TestPigServer.java
pig/trunk/test/org/apache/pig/test/TestProjectRange.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1598445&r1=1598444&r2=1598445&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 30 00:28:29 2014
@@ -159,6 +159,8 @@ PIG-3882: Multiquery off mode execution
BUG FIXES
+PIG-3967: Grunt fail if we running more statement after first store (daijy)
+
PIG-3915: MapReduce queries in Pigmix outputs different results than Pig's (keren3000 via daijy)
PIG-3955: Remove url.openStream() file descriptor leak from JCC (aniket486)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1598445&r1=1598444&r2=1598445&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Fri May 30 00:28:29 2014
@@ -1677,12 +1677,12 @@ public class PigServer {
}
}
- void validateQuery() throws FrontendException {
+ private void validateQuery() throws FrontendException {
String query = buildQuery();
QueryParserDriver parserDriver = new QueryParserDriver( pigContext, scope, fileNameMap );
try {
LogicalPlan plan = parserDriver.parse( query );
- plan.validate(pigContext, scope);
+ plan.validate(pigContext, scope, true);
} catch(FrontendException ex) {
scriptCache.remove( scriptCache.size() -1 );
throw ex;
@@ -1741,7 +1741,7 @@ public class PigServer {
}
private void compile() throws IOException {
- lp.validate(pigContext, scope);
+ lp.validate(pigContext, scope, false);
currDAG.postProcess();
}
Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java?rev=1598445&r1=1598444&r2=1598445&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java (original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalPlan.java Fri May 30 00:28:29 2014
@@ -165,8 +165,9 @@ public class LogicalPlan extends BaseOpe
return Integer.toString(hos.getHashCode().asInt());
}
-
- public void validate(PigContext pigContext, String scope) throws FrontendException {
+
+ public void validate(PigContext pigContext, String scope, boolean skipInputOutputValidation)
+ throws FrontendException {
new DanglingNestedNodeRemover(this).visit();
new ColumnAliasConversionVisitor(this).visit();
@@ -204,7 +205,7 @@ public class LogicalPlan extends BaseOpe
// compute whether output data is sorted or not
new SortInfoSetter(this).visit();
- if (!(pigContext.inExplain || pigContext.inDumpSchema)) {
+ if (!(skipInputOutputValidation || pigContext.inExplain || pigContext.inDumpSchema)) {
// Validate input/output file
new InputOutputFileValidatorVisitor(this, pigContext).visit();
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=1598445&r1=1598444&r2=1598445&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServer.java Fri May 30 00:28:29 2014
@@ -915,6 +915,33 @@ public class TestPigServer {
_testSkipParseInRegisterForBatch(true, 7, 1);
}
+ @Test
+ // See PIG-3967
+ public void testGruntValidation() throws IOException {
+ PigServer pigServer = new PigServer(ExecType.LOCAL);
+ Data data = resetData(pigServer);
+
+ data.set("foo",
+ tuple("a", 1, "b"),
+ tuple("b", 2, "c"),
+ tuple("c", 3, "d"));
+
+ pigServer.setValidateEachStatement(true);
+ pigServer.registerQuery("A = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
+ pigServer.registerQuery("store A into '" + tempDir + "/testGruntValidation1';");
+ pigServer.registerQuery("B = LOAD 'foo' USING mock.Storage() AS (f1:chararray,f2:int,f3:chararray);");
+ pigServer.registerQuery("store B into '" + tempDir + "/testGruntValidation2';"); // This should pass
+ boolean validationExceptionCaptured = false;
+ try {
+ // This should fail due to output validation
+ pigServer.registerQuery("store A into '" + tempDir + "/testGruntValidation1';");
+ } catch (FrontendException e) {
+ validationExceptionCaptured = true;
+ }
+
+ assertTrue(validationExceptionCaptured);
+ }
+
private void _testSkipParseInRegisterForBatch(boolean skipParseInRegisterForBatch,
int numTimesInitiated, int numTimesSchemaCalled) throws Throwable {
MockTrackingStorage.numTimesInitiated = 0;
Modified: pig/trunk/test/org/apache/pig/test/TestProjectRange.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestProjectRange.java?rev=1598445&r1=1598444&r2=1598445&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestProjectRange.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestProjectRange.java Fri May 30 00:28:29 2014
@@ -784,7 +784,7 @@ public class TestProjectRange {
private LogicalPlan createAndProcessLPlan(String query) throws FrontendException {
//TODO: create a common util function for logical plan tests
LogicalPlan lp = generateLogicalPlan(query);
- lp.validate(pigServer.getPigContext(), "test");
+ lp.validate(pigServer.getPigContext(), "test", false);
return lp;
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1598445&r1=1598444&r2=1598445&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Fri May 30 00:28:29 2014
@@ -1068,7 +1068,7 @@ public class Util {
QueryParserDriver parserDriver = new QueryParserDriver( pc, "test", fileNameMap );
org.apache.pig.newplan.logical.relational.LogicalPlan lp = parserDriver.parse( query );
- lp.validate(pc, "test");
+ lp.validate(pc, "test", false);
return lp;
}
@@ -1077,7 +1077,7 @@ public class Util {
QueryParserDriver parserDriver = new QueryParserDriver( pc, "test", fileNameMap );
org.apache.pig.newplan.logical.relational.LogicalPlan lp = parserDriver.parse( query );
- lp.validate(pc, "test");
+ lp.validate(pc, "test", false);
return lp;
}