You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by kn...@apache.org on 2013/09/25 20:24:37 UTC
svn commit: r1526257 - in /pig/branches/branch-0.12: CHANGES.txt
src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java
test/org/apache/pig/test/TestMultiQueryBasic.java
Author: knoguchi
Date: Wed Sep 25 18:24:37 2013
New Revision: 1526257
URL: http://svn.apache.org/r1526257
Log:
PIG-3458: ScalarExpression lost with multiquery optimization (knoguchi)
Modified:
pig/branches/branch-0.12/CHANGES.txt
pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java
pig/branches/branch-0.12/test/org/apache/pig/test/TestMultiQueryBasic.java
Modified: pig/branches/branch-0.12/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/CHANGES.txt?rev=1526257&r1=1526256&r2=1526257&view=diff
==============================================================================
--- pig/branches/branch-0.12/CHANGES.txt (original)
+++ pig/branches/branch-0.12/CHANGES.txt Wed Sep 25 18:24:37 2013
@@ -241,6 +241,8 @@ PIG-3013: BinInterSedes improve chararra
BUG FIXES
+PIG-3458: ScalarExpression lost with multiquery optimization (knoguchi)
+
PIG-3360: Some intermittent negative e2e tests fail on hadoop 2 (daijy)
PIG-3468: PIG-3123 breaks e2e test Jython_Diagnostics_2 (daijy)
Modified: pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java?rev=1526257&r1=1526256&r2=1526257&view=diff
==============================================================================
--- pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java (original)
+++ pig/branches/branch-0.12/src/org/apache/pig/newplan/logical/visitor/ScalarVisitor.java Wed Sep 25 18:24:37 2013
@@ -71,9 +71,13 @@ public class ScalarVisitor extends AllEx
LogicalPlan lp = (LogicalPlan) attachedOp.getPlan();
List<Operator> succs = lp.getSuccessors( refOp );
LOStore store = null;
+ FuncSpec interStorageFuncSpec = new FuncSpec(InterStorage.class.getName());
if( succs != null ) {
for( Operator succ : succs ) {
- if( succ instanceof LOStore ) {
+ if( succ instanceof LOStore
+ && ((LOStore)succ).isTmpStore()
+ && interStorageFuncSpec.equals(
+ ((LOStore)succ).getOutputSpec().getFuncSpec() ) ) {
store = (LOStore)succ;
break;
}
@@ -81,14 +85,13 @@ public class ScalarVisitor extends AllEx
}
if( store == null ) {
- FuncSpec funcSpec = new FuncSpec(InterStorage.class.getName());
FileSpec fileSpec;
try {
- fileSpec = new FileSpec( FileLocalizer.getTemporaryPath( pigContext ).toString(), funcSpec ); // TODO: need to hookup the pigcontext.
+ fileSpec = new FileSpec( FileLocalizer.getTemporaryPath( pigContext ).toString(), interStorageFuncSpec ); // TODO: need to hookup the pigcontext.
} catch (IOException e) {
throw new PlanValidationException( expr, "Failed to process scalar" + e);
}
- StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(funcSpec);
+ StoreFuncInterface stoFunc = (StoreFuncInterface)PigContext.instantiateFuncFromSpec(interStorageFuncSpec);
String sig = LogicalPlanBuilder.newOperatorKey(scope);
stoFunc.setStoreFuncUDFContextSignature(sig);
store = new LOStore(lp, fileSpec, stoFunc, sig);
Modified: pig/branches/branch-0.12/test/org/apache/pig/test/TestMultiQueryBasic.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.12/test/org/apache/pig/test/TestMultiQueryBasic.java?rev=1526257&r1=1526256&r2=1526257&view=diff
==============================================================================
--- pig/branches/branch-0.12/test/org/apache/pig/test/TestMultiQueryBasic.java (original)
+++ pig/branches/branch-0.12/test/org/apache/pig/test/TestMultiQueryBasic.java Wed Sep 25 18:24:37 2013
@@ -577,6 +577,44 @@ public class TestMultiQueryBasic {
Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), "output1_checkOutputSpec_test");
Util.deleteFile(new PigContext(ExecType.LOCAL, new Properties()), "output2_checkOutputSpec_test");
}
+
+ /**
+ * Test that MultiQuery optimization won't use user's output for
+ * ScalarExpression (and get empty output) at the end
+ */
+ @Test
+ public void testMultiQueryWithScalarExpression() throws Exception {
+
+ System.out.println("===== multi-query with ScalarExpression =====");
+
+ String[] inputData = {"john","henry", "adam"};
+ Util.createLocalInputFile("queryInput.txt", inputData);
+
+ myPig.setBatchOn();
+
+ myPig.registerQuery("a = load 'queryInput.txt' using PigStorage() as (uname:chararray);");
+ myPig.registerQuery("b = group a ALL;");
+ myPig.registerQuery("c = foreach b generate COUNT(a) as count;");
+ myPig.registerQuery("store c into 'output1';");
+ myPig.registerQuery("z = load 'queryInput.txt' using PigStorage() as (uname:chararray);");
+ myPig.registerQuery("y = foreach z generate uname, c.count;");
+ myPig.registerQuery("store y into 'output2';");
+
+ List<ExecJob> jobs = myPig.executeBatch();
+
+ for (ExecJob job : jobs) {
+ assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
+ }
+ myPig.registerQuery("aa = load 'output2' as (uname:chararray, cnt:int) ;");
+ Iterator<Tuple> it = myPig.openIterator("aa");
+ int i = 0;
+ while(it.hasNext()) {
+ Tuple t = it.next();
+ i++;
+ assertEquals(3, t.get(1));
+ }
+ assertEquals(3, i);
+ }
private static final String DUMMY_STORE_WITH_OUTPUTFORMAT_CLASS
= "org.apache.pig.test.TestMultiQueryBasic\\$DummyStoreWithOutputFormat";