You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2008/09/04 23:25:43 UTC
svn commit: r692253 [3/3] - in /incubator/pig/branches/types: ./
src/org/apache/pig/
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/
src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/
src/org/apache/pig/backend/hadoop/ex...
Modified: incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj (original)
+++ incubator/pig/branches/types/src/org/apache/pig/tools/pigscript/parser/PigScriptParser.jj Thu Sep 4 14:25:41 2008
@@ -188,6 +188,7 @@
<PIG_START> MORE :
{
<"'"> {prevState = PIG_START;} : IN_STRING
+| <"`"> {prevState = PIG_START;} : IN_COMMAND
| <["A","a"]["S","s"]> {prevState = PIG_START;} : SCHEMA_DEFINITION
| <["G","g"]["E","e"]["N","n"]["E","e"]["R","r"]["A","a"]["T","t"]["E","e"]> {prevState = PIG_START;} : GENERATE
| <"{"> {pigBlockLevel = 1;} : IN_BLOCK
@@ -220,6 +221,14 @@
| <(~[])>
}
+<IN_COMMAND> MORE :
+{
+ <"\\`">
+| <"`"> { SwitchTo(prevState);}
+| <("\n" | "\r" | "\r\n")> {secondary_prompt();}
+| <(~[])>
+}
+
<GENERATE> MORE :
{
<"{">
@@ -282,6 +291,7 @@
| <"{"> {pigBlockLevel++;}
| <"}"(";")?> {pigBlockLevel--; if (pigBlockLevel == 0) SwitchTo(PIG_END);}
| <"'"> {prevState = IN_BLOCK;} : IN_STRING
+| <"`"> {prevState = IN_BLOCK;} : IN_COMMAND
| <"--"> {prevState = IN_BLOCK;} : SINGLE_LINE_COMMENT
| <"/*"> {prevState = IN_BLOCK;} : MULTI_LINE_COMMENT
| <("\n" | "\r" | "\r\n")> {secondary_prompt();}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestInputOutputFileValidator.java Thu Sep 4 14:25:41 2008
@@ -115,7 +115,7 @@
new FileSpec(inputFile, new FuncSpec("org.apache.pig.builtin.PigStorage")) ;
FileSpec filespec2 =
new FileSpec(outputFile, new FuncSpec("org.apache.pig.builtin.PigStorage"));
- LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null) ;
+ LOLoad load = new LOLoad(plan, genNewOperatorKeyId(), filespec1, null, true) ;
LOStore store = new LOStore(plan, genNewOperatorKeyId(), filespec2) ;
plan.add(load) ;
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestJobSubmission.java Thu Sep 4 14:25:41 2008
@@ -153,7 +153,7 @@
FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
POStore st = new POStore(new OperatorKey("", r.nextLong()));
ld.setPc(pc);
ld.setLFile(LFSpec);
@@ -200,7 +200,7 @@
FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
POStore st = new POStore(new OperatorKey("", r.nextLong()));
ld.setPc(pc);
ld.setLFile(LFSpec);
@@ -247,7 +247,7 @@
FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName()));
FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
POStore st = new POStore(new OperatorKey("", r.nextLong()));
ld.setPc(pc);
ld.setLFile(LFSpec);
@@ -309,7 +309,7 @@
FileSpec LFSpec = new FileSpec(hadoopLdFile,new FuncSpec(PigStorage.class.getName()));
FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
POStore st = new POStore(new OperatorKey("", r.nextLong()));
ld.setPc(pc);
ld.setLFile(LFSpec);
@@ -376,7 +376,7 @@
FileSpec LFSpec = new FileSpec(hadoopLdFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
FileSpec SFSpec = new FileSpec(stFile,new FuncSpec(PigStorage.class.getName()));
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
POStore st = new POStore(new OperatorKey("", r.nextLong()));
ld.setPc(pc);
ld.setLFile(LFSpec);
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestLocalJobSubmission.java Thu Sep 4 14:25:41 2008
@@ -373,7 +373,7 @@
FileSpec LFSpec = new FileSpec(ldFile, new FuncSpec(PigStorage.class.getName(), new String[]{","}));
FileSpec SFSpec = new FileSpec(stFile, new FuncSpec(PigStorage.class.getName()));
- POLoad ld = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ld = new POLoad(new OperatorKey("", r.nextLong()), true);
POStore st = new POStore(new OperatorKey("", r.nextLong()));
ld.setPc(pc);
ld.setLFile(LFSpec);
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidator.java Thu Sep 4 14:25:41 2008
@@ -896,11 +896,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1020,11 +1020,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1192,7 +1192,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1260,7 +1260,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1330,7 +1330,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1400,7 +1400,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1480,7 +1480,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1578,7 +1578,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1673,7 +1673,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1801,7 +1801,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -1905,11 +1905,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -2076,11 +2076,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
@@ -2215,11 +2215,11 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
LOLoad load2 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// schema for input#1
Schema inputSchema1 = null ;
Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java Thu Sep 4 14:25:41 2008
@@ -445,7 +445,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// set schemas
load1.setEnforcedSchema(null) ;
@@ -511,7 +511,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
// set schemas
load1.setEnforcedSchema(null) ;
Modified: incubator/pig/branches/types/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/Util.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/Util.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/Util.java Thu Sep 4 14:25:41 2008
@@ -17,9 +17,14 @@
*/
package org.apache.pig.test;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
+import static java.util.regex.Matcher.quoteReplacement;
+import junit.framework.Assert;
import org.apache.pig.backend.executionengine.ExecException;
import org.apache.pig.data.*;
@@ -145,6 +150,45 @@
return t;
}
+ /**
+ * Helper to create a temporary file with given input data for use in test cases.
+ *
+ * @param tmpFilenamePrefix file-name prefix
+ * @param tmpFilenameSuffix file-name suffix
+ * @param inputData input for test cases, each string in inputData[] is written
+ * on one line
+ * @return {@link File} handle to the created temporary file
+ * @throws IOException
+ */
+ static public File createInputFile(String tmpFilenamePrefix,
+ String tmpFilenameSuffix,
+ String[] inputData)
+ throws IOException {
+ File f = File.createTempFile(tmpFilenamePrefix, tmpFilenameSuffix);
+ PrintWriter pw = new PrintWriter(f);
+ for (int i=0; i<inputData.length; i++){
+ pw.println(inputData[i]);
+ }
+ pw.close();
+ return f;
+ }
+
+ /**
+ * Helper function to check if the result of a Pig Query is in line with
+ * expected results.
+ *
+ * @param actualResults Result of the executed Pig query
+ * @param expectedResults Expected results to validate against
+ */
+ static public void checkQueryOutputs(Iterator<Tuple> actualResults,
+ Tuple[] expectedResults) {
+
+ for (Tuple expected : expectedResults) {
+ Tuple actual = actualResults.next();
+ Assert.assertEquals(expected.toString(), actual.toString());
+ }
+ }
+
static public void printQueryOutput(Iterator<Tuple> actualResults,
Tuple[] expectedResults) {
@@ -160,4 +204,18 @@
}
System.out.println("---End----") ;
}
+
+ /**
+ * Helper method to replace all occurrences of "\" with "\\" in a
+ * string. This is useful to fix the file path string on Windows
+ * where "\" is used as the path separator.
+ *
+ * @param str Any string
+ * @return The resulting string
+ */
+ public static String encodeEscape(String str) {
+ String regex = "\\\\";
+ String replacement = quoteReplacement("\\\\");
+ return str.replaceAll(regex, replacement);
+ }
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/GenPhyOp.java Thu Sep 4 14:25:41 2008
@@ -568,7 +568,7 @@
public static POLoad topLoadOp() {
- POLoad ret = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ret = new POLoad(new OperatorKey("", r.nextLong()), true);
ret.setPc(pc);
return ret;
}
Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/TestHelper.java Thu Sep 4 14:25:41 2008
@@ -170,11 +170,11 @@
public static boolean areFilesSame(FileSpec expLocal, FileSpec actHadoop, PigContext pc) throws ExecException, IOException{
Random r = new Random();
- POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ldExp = new POLoad(new OperatorKey("", r.nextLong()), true);
ldExp.setPc(pc);
ldExp.setLFile(expLocal);
- POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong()));
+ POLoad ldAct = new POLoad(new OperatorKey("", r.nextLong()), true);
ldAct.setPc(pc);
ldAct.setLFile(actHadoop);
Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java Thu Sep 4 14:25:41 2008
@@ -42,7 +42,7 @@
LOLoad load = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null) ;
+ null, true) ;
return load ;
} catch (IOException e) {
throw new AssertionError("This cannot happen") ;
Modified: incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java
URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java?rev=692253&r1=692252&r2=692253&view=diff
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java (original)
+++ incubator/pig/branches/types/test/org/apache/pig/test/utils/dotGraph/LogicalPlanLoader.java Thu Sep 4 14:25:41 2008
@@ -94,7 +94,7 @@
FileSpec fileSpec = new FileSpec("pi",
new FuncSpec(PigStorage.class.getName())) ;
try {
- load = new LOLoad(plan, getKey(node.attributes), fileSpec, null) ;
+ load = new LOLoad(plan, getKey(node.attributes), fileSpec, null, true) ;
fillSchema(load, node.attributes) ;
}
catch (IOException ioe) {