You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ol...@apache.org on 2009/11/24 18:10:49 UTC
svn commit: r883790 - in /hadoop/pig/trunk: ./
src/org/apache/pig/tools/parameters/ test/org/apache/pig/test/
test/org/apache/pig/test/data/
Author: olga
Date: Tue Nov 24 17:10:47 2009
New Revision: 883790
URL: http://svn.apache.org/viewvc?rev=883790&view=rev
Log:
PIG-598: Parameter substitution () should not be performed in
comments (thejas via olgan)
Added:
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig
hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Tue Nov 24 17:10:47 2009
@@ -36,6 +36,9 @@
BUG FIXES
+PIG-598: Parameter substitution ($PARAMETER) should not be performed in
+comments (thejas via olgan)
+
PIG-1064: Behaviour of COGROUP with and without schema when using "*" operator
(pradeepkth)
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParamLoader.jj Tue Nov 24 17:10:47 2009
@@ -32,6 +32,11 @@
import java.io.IOException;
import java.util.Hashtable;
+//warnings in by code generated by javacc cannot be fixed here,
+// so suppressing all warnings for this class. But this does not help in
+//supressing Warnings in other classes generated by this .jj file
+@SuppressWarnings("all")
+
public class ParamLoader {
private PreprocessorContext pc;
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/ParameterSubstitutionPreprocessor.java Tue Nov 24 17:10:47 2009
@@ -52,7 +52,6 @@
paramParser = new ParamLoader(sr);
paramParser.setContext(pc);
-
pigParser = new PigFileParser(sr);
pigParser.setContext(pc);
}
@@ -87,33 +86,10 @@
}
private void parsePigFile(BufferedReader in, Writer out) throws ParseException {
-
+ pigParser.setOutputWriter(out);
+ pigParser.ReInit(in);
try {
- String line;
- int lineNum=0;
-
- while((line = in.readLine())!= null)
- {
- lineNum++;
-
- if (line.length() == 0) {
- out.append("\n");
- continue;
- }
-
- if ( !(line.startsWith("%")) ) {
- //process an ordinary pig line - perform substitution
- String sub_line = pc.substitute(line);
- out.append(sub_line);
- out.append("\n");
- continue;
- }
-
- //parse the declare/default line
- pigParser.ReInit(new StringReader(line));
- pigParser.Parse();
- }
-
+ pigParser.Parse();
//close input and output streams
in.close();
out.flush();
@@ -122,7 +98,6 @@
RuntimeException rte = new RuntimeException(e.getMessage() , e);
throw rte;
}
-
}
/*
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PigFileParser.jj Tue Nov 24 17:10:47 2009
@@ -18,7 +18,7 @@
/**
* This grammar is used to process %declare and %default commands
- * within pig script
+ * within pig script, and substitute parameter values
*/
options {
@@ -26,20 +26,34 @@
STATIC = false;
IGNORE_CASE = true;
JAVA_UNICODE_ESCAPE = true;
+ //DEBUG_PARSER = true;
+ LOOKAHEAD = 2;
+
}
PARSER_BEGIN(PigFileParser)
+
package org.apache.pig.tools.parameters;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Hashtable;
+import java.io.Writer;
+import java.lang.StringBuilder;
+
+//warnings in by code generated by javacc cannot be fixed here,
+// so suppressing all warnings for this class. But this does not help in
+//supressing Warnings in other classes generated by this .jj file
+@SuppressWarnings("all")
public class PigFileParser {
private PreprocessorContext pc;
+ private Writer out;
public void setContext(PreprocessorContext pc) {
this.pc = pc;
}
-
+ public void setOutputWriter(Writer out) {
+ this.out = out;
+ }
private static String unquote(String s)
{
if (s.charAt(0) == '\'' && s.charAt(s.length()-1) == '\'')
@@ -53,12 +67,20 @@
PARSER_END(PigFileParser)
-SKIP :
+TOKEN :
{
- "\n"
-| "\r"
-| " "
-| "\t"
+ <NEWLINE: "\n" | "\r">
+ |
+ <SPACE: " " | "\t">
+}
+
+// comments(single line and multi-line)
+TOKEN :
+{
+ <COMMENT: "--"(~["\r","\n"])*
+ | "#!" (~["\r","\n"])*
+ | "/*" (~["*"])* "*" ("*" | (~["*","/"] (~["*"])* "*"))* "/"
+ >
}
TOKEN:
@@ -68,13 +90,15 @@
<#DIGIT : ["0"-"9"] >
|
<#SPECIALCHAR : ["_"] >
+ |
+ <#DOLLAR : ["$"]>
}
TOKEN :
{
<DECLARE: "%declare" >
|
- <PIGDEFAULT: "%default" >
+ <PIGDEFAULT: "%default" >
}
TOKEN :
@@ -85,30 +109,75 @@
|
<SHELLCMD: "`" (~["`"])* "`" >
|
- <OTHER: ~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r", "%"] (~["\n","\r"])* >
+ // see others() rule for use of OTHER and NOT_OTHER_CHAR
+ // others() is supposed to match 'everything else'. To ensure that others()
+ // don't swallow other(all the ones above) tokens, it uses two tokens OTHER and NOT_OTHER_CHAR
+ // NOT_OTHER_CHAR consists of first characters of other tokens, and OTHER consists of one
+ // or more chars that don't belong to NOT_OTHER_CHAR. Since the tokeniser matches the longest
+ // match, other tokens will get matched instead of NOT_OTHER_CHAR.
+ <OTHER: (~["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r", "%", "/", "-", "$"])+ >
+ |
+ <NOT_OTHER_CHAR: ["\"" , "'" , "`" , "a"-"z" , "A"-"Z" , "_" , "#" , "=" , " " , "\n" , "\t" , "\r", "%", "/", "-", "$"] >
+
}
+void Parse() throws IOException : {}
+{
+ (input())*<EOF>
+}
-
-void Parse() throws IOException :
+void input() throws IOException :
{
- Token id, val;
- Boolean overwrite;
String s;
+ Token strTok;
}
{
- <DECLARE> { overwrite=true; }
+ <DECLARE>
(
- id=<IDENTIFIER>
+ param_value(true) // overwrite=true
+ )
+ |
+ <PIGDEFAULT>
+ (
+ param_value(false) // overwrite=false
+ )
+ |
+ s = paramString(){}
+ {
+ //process an ordinary pig line - perform substitution
+ String sub_line = pc.substitute(s);
+ out.append(sub_line );
+ }
+ |
+ strTok = <IDENTIFIER>{}
+ {
+ out.append(strTok.image );
+ }
+ |
+ write_ignore_toks()
+
+}
+
+void param_value(boolean overwrite) throws IOException:
+{
+ Token id, val;
+ String s;
+ String other;
+ Token strTok;
+}
+{
+ (ignore_toks_nonewline())*
+ id=<IDENTIFIER>
+ (ignore_toks_nonewline())*
(
- val=<OTHER> { pc.processOrdLine(id.image , val.image);}
+ s=others() (ignore_toks_nonewline())* write_newline() { pc.processOrdLine(id.image , s, overwrite);}
|
- val=<IDENTIFIER>
+ val=<IDENTIFIER> // this construct is for cases like a.123
{s = val.image;}
- (
- val =<OTHER>
- {s += val.image;}
+ ( LOOKAHEAD( 2 )
+ other = others() (ignore_toks_nonewline())* write_newline()
+ { s += other; }
)?
{pc.processOrdLine(id.image , s, overwrite);}
|
@@ -117,25 +186,136 @@
val=<LITERAL> { s = unquote(val.image); pc.processOrdLine(id.image, s, overwrite); }
)
- )
- |
- <PIGDEFAULT> { overwrite=false; }
+}
+
+//match others, see comments above
+//on description of OTHER , NOT_OTHER_CHAR
+String others() throws IOException :
+{ Token t;
+ StringBuilder sb = new StringBuilder();
+}
+{
(
- id=<IDENTIFIER>
- (
- val=<OTHER> { pc.processOrdLine(id.image , val.image);}
- |
- val=<IDENTIFIER>
- {s = val.image;}
- (
- val =<OTHER>
- {s += val.image;}
- )?
- {pc.processOrdLine(id.image , s, overwrite);}
- |
- val=<SHELLCMD> { pc.processShellCmd(id.image , val.image , overwrite); }
- |
- val=<LITERAL> { s = unquote(val.image); pc.processOrdLine(id.image , s, overwrite); }
- )
+ t=<OTHER>
+ {
+ sb.append(t.image);
+ }
+ |
+ t=<NOT_OTHER_CHAR>
+ {
+ sb.append(t.image);
+ }
)
+ (
+ t=<OTHER>
+ {
+ sb.append(t.image);
+ }
+ |
+ t=<NOT_OTHER_CHAR>
+ {
+ sb.append(t.image);
+ }
+ |
+ t=<IDENTIFIER>
+ {
+ //eg of this match is a unquoted filename - /d1/abc/
+ sb.append(t.image);
+ }
+ |
+ t=<LITERAL>
+ {
+ sb.append(t.image);
+ }
+ |
+ t=<SHELLCMD>
+ {
+ sb.append(t.image);
+ }
+ )*{
+ return sb.toString();
+ }
+}
+
+
+// a string that can contain parameter
+String paramString() throws IOException :
+{Token t;
+ String str;
+}
+{
+ str = others()
+ {
+ return str;
+ }
+ |
+ t=<LITERAL>
+ {
+ return t.image;
+ }
+ |
+ t = <SHELLCMD>{}
+ {
+ return t.image;
+ }
+}
+
+// write the newlines,spaces and comments to preserve formatting
+void write_ignore_toks() throws IOException :
+{ String str;}
+{
+ str = ignore_toks(){
+ out.append(str);
+ }
+}
+
+
+// match the newlines,spaces and comments
+String ignore_toks_nonewline() throws IOException :
+{ Token t; }
+{
+ t = <COMMENT>{
+ return t.image;
+ }
+ |
+ t = <SPACE>{
+ return t.image;
+ }
+}
+
+
+// match the newlines,spaces and comments
+String ignore_toks() throws IOException :
+{ Token t;
+ String str;
}
+{
+ t = <COMMENT>{
+ return t.image;
+ }
+ |
+ str = space_or_newline(){
+ return str;
+ }
+}
+
+String space_or_newline() :
+{ Token t; }
+{
+ t = <SPACE>{
+ return t.image;
+ }
+ |
+ t = <NEWLINE>{
+ return t.image;
+ }
+}
+
+void write_newline() throws IOException:
+{ Token t; }
+{
+ t = <NEWLINE>{
+ out.append(t.image);
+ }
+}
+
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/parameters/PreprocessorContext.java Tue Nov 24 17:10:47 2009
@@ -210,16 +210,16 @@
return streamData.trim();
}
- private static String id_regex = "\\$[_]*[a-zA-Z][a-zA-Z_0-9]*";
+ private Pattern id_pattern = Pattern.compile("\\$[_]*[a-zA-Z][a-zA-Z_0-9]*");
+
public String substitute(String line) {
int index = line.indexOf('$');
if (index == -1) return line;
String replaced_line = line;
-
- Pattern identifier = Pattern.compile( id_regex );
- Matcher keyMatcher = identifier.matcher( line );
+
+ Matcher keyMatcher = id_pattern.matcher( line );
String key="";
String val="";
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Tue Nov 24 17:10:47 2009
@@ -167,7 +167,7 @@
ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResultDefault.pig");
BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
@@ -259,7 +259,7 @@
ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
@@ -310,7 +310,7 @@
ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
@@ -359,7 +359,7 @@
ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResultCmdLnPriorDeclare.pig");
BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
@@ -409,7 +409,7 @@
ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResult4.pig");
BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
@@ -913,7 +913,7 @@
ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
- pigExResultStream = new FileInputStream(basedir + "/ExpectedResult.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResultMulDecs.pig");
BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
@@ -1336,4 +1336,51 @@
log.info("Done");
}
+
+ /* Test case 25
+ * Test that params in comments are untouched
+ */
+ @Test
+ public void testCommentWithParam() throws Exception{
+ try {
+ ParameterSubstitutionPreprocessor ps = new ParameterSubstitutionPreprocessor(50);
+ pigIStream = new BufferedReader(new FileReader(basedir + "/inputComment.pig"));
+ pigOStream = new FileWriter(basedir + "/output1.pig");
+
+ String[] arg = {"date='20080228'"};
+ String[] argFiles = null;
+ ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+ FileInputStream pigResultStream = new FileInputStream(basedir + "/output1.pig");
+ pigExResultStream = new FileInputStream(basedir + "/ExpectedResultComment.pig");
+ BufferedReader inExpected = new BufferedReader(new InputStreamReader(pigExResultStream));
+ BufferedReader inResult = new BufferedReader(new InputStreamReader(pigResultStream));
+
+ String exLine;
+ String resLine;
+ int lineNum=0;
+
+ while (true) {
+ lineNum++;
+ exLine = inExpected.readLine();
+ resLine = inResult.readLine();
+ if (exLine==null || resLine==null)
+ break;
+ assertEquals("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum ,exLine.trim(), resLine.trim());
+ }
+ if (!(exLine==null && resLine==null)) {
+ fail ("Expected : "+exLine+" , but got : "+resLine+" in line num : "+lineNum);
+ }
+
+ inExpected.close();
+ inResult.close();
+ } catch (ParseException e) {
+ fail ("Got ParseException : " + e.getMessage());
+ } catch (RuntimeException e) {
+ fail ("Got RuntimeException : " + e.getMessage());
+ } catch (Error e) {
+ fail ("Got error : " + e.getMessage());
+ }
+
+ }
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult.pig Tue Nov 24 17:10:47 2009
@@ -8,4 +8,4 @@
store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
grpInactiveAcct = group inactiveAccounts all;
countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
-store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult2.pig Tue Nov 24 17:10:47 2009
@@ -8,4 +8,4 @@
store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
grpInactiveAcct = group inactiveAccounts all;
countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
-store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResult4.pig Tue Nov 24 17:10:47 2009
@@ -0,0 +1,14 @@
+
+
+
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig Tue Nov 24 17:10:47 2009
@@ -0,0 +1,12 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultComment.pig Tue Nov 24 17:10:47 2009
@@ -0,0 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
+
+/* comment */
+
+/* l = load '$zxv';
+ store l into '$zxcv';
+*/
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate /* $col 1 */ $0,$12,$7;
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- testing $comment
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultDefault.pig Tue Nov 24 17:10:47 2009
@@ -0,0 +1,12 @@
+
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig Tue Nov 24 17:10:47 2009
@@ -0,0 +1,13 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/input5.pig Tue Nov 24 17:10:47 2009
@@ -1,5 +1,4 @@
%declare udfs /data/pigudf.jar
-
register $udfs;
aa = load '$loadfile ' using PigStorage('\x01');
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig?rev=883790&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputComment.pig Tue Nov 24 17:10:47 2009
@@ -0,0 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
+%default date /* comment */ 1000 -- setting default
+/* comment */ %declare arity 16
+
+/* l = load '$zxv';
+ store l into '$zxcv';
+*/
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '$arity') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate /* $col 1 */ $0,$12,$7;
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- testing $comment
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputDefault.pig Tue Nov 24 17:10:47 2009
@@ -1,5 +1,4 @@
%default date '20080228'
-
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
a = foreach bb generate $0,$12,$7;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputEscape.pig Tue Nov 24 17:10:47 2009
@@ -1,5 +1,4 @@
%default column1 'asdf'
-
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
a = foreach bb generate $0,$12,$7;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig Tue Nov 24 17:10:47 2009
@@ -1,4 +1,5 @@
+
%DECLARE date '20080228'
%DECLARE formatted_date `sh test/org/apache/pig/test/data/generate_date.sh $date`
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$formatted_date' using PigStorage('\x01');
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/inputThreeParams.pig Tue Nov 24 17:10:47 2009
@@ -1,9 +1,7 @@
%default tableName 'value to be skipped'
-
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
a = foreach bb generate $0,$12,$7;
-
%declare tableName 'inactiveAccounts'
--generate inactive accts
$tableName = filter a by ($1 neq '') and ($1 == '2') parallel 400;
Modified: hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig?rev=883790&r1=883789&r2=883790&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/data/output1.pig Tue Nov 24 17:10:47 2009
@@ -1,7 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
-register /data/pigudf.jar;
+/* comment */
-aa = load '/user/pig/tests/data/singlefile/textdoc.txt ' using PigStorage('\x01');
+/* l = load '$zxv';
+ store l into '$zxcv';
+*/
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
-a = foreach bb generate $0,$12,$7;
-store inactiveAccounts into '/user/pig/tests/results/DefineClause_4.out';
+a = foreach bb generate /* $col 1 */ $0,$12,$7;
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- testing $comment
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';