You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2009/12/17 19:36:36 UTC
svn commit: r891828 [15/15] - in /hadoop/pig/branches/load-store-redesign:
./ contrib/
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/string/
contrib/...
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestRelevantFields.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestRelevantFields.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestRelevantFields.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestRelevantFields.java Thu Dec 17 18:36:28 2009
@@ -52,7 +52,7 @@
LogicalPlanTester planTester = new LogicalPlanTester();
@Test
- public void testQueryForeach1() {
+ public void testQueryForeach1() throws FrontendException {
String query = "foreach (load 'a') generate $1,$2;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -72,7 +72,7 @@
}
@Test
- public void testQueryForeach2() {
+ public void testQueryForeach2() throws FrontendException {
String query = "foreach (load 'a') generate $1,$2+$3;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -91,7 +91,7 @@
}
@Test
- public void testQueryForeach3() {
+ public void testQueryForeach3() throws FrontendException {
String query = "foreach (load 'a') generate $1,CONCAT($2,$3);";
LogicalPlan lp = planTester.buildPlan(query);
@@ -110,7 +110,7 @@
}
@Test
- public void testQueryCogroup1() {
+ public void testQueryCogroup1() throws FrontendException {
String query = "foreach (cogroup (load 'a') by $1, (load 'b') by $1) generate org.apache.pig.builtin.AVG($1) ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -142,7 +142,7 @@
}
@Test
- public void testQueryGroupAll() throws Exception {
+ public void testQueryGroupAll() throws FrontendException {
String query = "foreach (group (load 'a') ALL) generate $1 ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -159,7 +159,7 @@
}
@Test
- public void testQueryGroup2() {
+ public void testQueryGroup2() throws FrontendException {
String query = "foreach (group (load 'a') by $1) generate group, '1' ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -177,7 +177,7 @@
}
@Test
- public void testQueryCogroup2() {
+ public void testQueryCogroup2() throws FrontendException {
String query = "foreach (cogroup (load 'a') by ($1), (load 'b') by ($1)) generate $1.$1, $2.$1 ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -205,7 +205,7 @@
}
@Test
- public void testQueryGroup3() {
+ public void testQueryGroup3() throws FrontendException {
String query = "foreach (group (load 'a') by ($6, $7)) generate flatten(group) ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -222,7 +222,7 @@
assertTrue(cogroupRelevantFields1.getNeedNoFields() == false);
}
- public void testQueryFilterNoSchema() {
+ public void testQueryFilterNoSchema() throws FrontendException {
planTester.buildPlan("a = load 'a';");
LogicalPlan lp = planTester.buildPlan("b = filter a by $1 == '3';");
@@ -235,15 +235,15 @@
}
@Test
- public void testQuerySplitNoSchema() {
+ public void testQuerySplitNoSchema() throws FrontendException {
planTester.buildPlan("a = load 'a';");
LogicalPlan lp = planTester.buildPlan("split a into b if $0 == '3', c if $1 == '3';");
LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
RequiredFields splitRelevantFields = split.getRelevantInputs(0, 0).get(0);
assertTrue(splitRelevantFields.needAllFields() == false);
- assertTrue(splitRelevantFields.needNoFields() == true);
- assertTrue(splitRelevantFields.getFields() == null);
+ assertTrue(splitRelevantFields.needNoFields() == false);
+ assertTrue(splitRelevantFields.getFields().size() == 1);
LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
RequiredFields splitbRelevantFields0 = splitb.getRelevantInputs(0, 0).get(0);
@@ -275,7 +275,7 @@
}
@Test
- public void testQueryOrderByNoSchema() {
+ public void testQueryOrderByNoSchema() throws FrontendException {
planTester.buildPlan("a = load 'a';");
LogicalPlan lp = planTester.buildPlan("b = order a by $1;");
@@ -294,7 +294,7 @@
}
@Test
- public void testQueryLimitNoSchema() {
+ public void testQueryLimitNoSchema() throws FrontendException {
planTester.buildPlan("a = load 'a';");
planTester.buildPlan("b = order a by $1;");
LogicalPlan lp = planTester.buildPlan("c = limit b 10;");
@@ -316,7 +316,7 @@
}
@Test
- public void testQueryDistinctNoSchema() {
+ public void testQueryDistinctNoSchema() throws FrontendException {
planTester.buildPlan("a = load 'a';");
LogicalPlan lp = planTester.buildPlan("b = distinct a;");
@@ -337,7 +337,7 @@
}
@Test
- public void testQueryStreamingNoSchema() {
+ public void testQueryStreamingNoSchema() throws FrontendException {
String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "`;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -349,7 +349,7 @@
}
@Test
- public void testQueryStreamingNoSchema1() {
+ public void testQueryStreamingNoSchema1() throws FrontendException {
String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -395,7 +395,7 @@
}
@Test
- public void testForeach5() {
+ public void testForeach5() throws FrontendException {
planTester.buildPlan("A = load 'a' AS (a1, a2, a3);");
planTester.buildPlan("B = load 'b' AS (b1, b2, b3, b4);");
planTester.buildPlan("C = cogroup A by ($1), B by ($1);");
@@ -436,7 +436,7 @@
}
@Test
- public void testQueryCrossNoSchema(){
+ public void testQueryCrossNoSchema() throws FrontendException {
String query = "c = cross (load 'a'), (load 'b');";
LogicalPlan lp = planTester.buildPlan(query);
@@ -445,7 +445,7 @@
}
@Test
- public void testQueryUnionNoSchema(){
+ public void testQueryUnionNoSchema() throws FrontendException {
String query = "c = union (load 'a'), (load 'b');";
LogicalPlan lp = planTester.buildPlan(query);
@@ -463,7 +463,7 @@
assertTrue(unionRelevantFields1.getFields().contains(new Pair<Integer, Integer>(1, 0)));
}
- public void testQueryFRJoinNoSchema(){
+ public void testQueryFRJoinNoSchema() throws FrontendException {
String query = "c = join (load 'a') by $0, (load 'b') by $0 using \"replicated\";";
LogicalPlan lp = planTester.buildPlan(query);
@@ -472,7 +472,7 @@
}
@Test
- public void testQueryJoinNoSchema(){
+ public void testQueryJoinNoSchema() throws FrontendException {
String query = "c = join (load 'a') by $0, (load 'b') by $0;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -481,7 +481,7 @@
}
@Test
- public void testQueryFilterWithSchema() {
+ public void testQueryFilterWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url,hitCount);");
LogicalPlan lp = planTester.buildPlan("b = filter a by $1 == '3';");
@@ -494,15 +494,24 @@
}
@Test
- public void testQuerySplitWithSchema() {
+ public void testQuerySplitWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url, hitCount);");
LogicalPlan lp = planTester.buildPlan("split a into b if url == '3', c if hitCount == '3';");
LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
- RequiredFields splitRelevantFields = split.getRelevantInputs(0, 0).get(0);
- assertTrue(splitRelevantFields.needAllFields() == false);
- assertTrue(splitRelevantFields.needNoFields() == true);
- assertTrue(splitRelevantFields.getFields() == null);
+ RequiredFields splitRelevantFields0 = split.getRelevantInputs(0, 0).get(0);
+ assertTrue(splitRelevantFields0.needAllFields() == false);
+ assertTrue(splitRelevantFields0.needNoFields() == false);
+ assertTrue(splitRelevantFields0.getFields().size() == 1);
+ assertTrue(splitRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
+
+ RequiredFields splitRelevantFields1 = split.getRelevantInputs(0, 1).get(0);
+ assertTrue(splitRelevantFields1.needAllFields() == false);
+ assertTrue(splitRelevantFields1.needNoFields() == false);
+ assertTrue(splitRelevantFields1.getFields().size() == 1);
+ assertTrue(splitRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
+
+ assertTrue(split.getRelevantInputs(0, 2) == null);
LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
RequiredFields splitbRelevantFields = splitb.getRelevantInputs(0, 1).get(0);
@@ -520,7 +529,7 @@
}
@Test
- public void testQueryOrderByWithSchema() {
+ public void testQueryOrderByWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url,hitCount);");
LogicalPlan lp = planTester.buildPlan("b = order a by $1;");
@@ -539,7 +548,7 @@
}
@Test
- public void testQueryLimitWithSchema() {
+ public void testQueryLimitWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url,hitCount);");
planTester.buildPlan("b = order a by $1;");
LogicalPlan lp = planTester.buildPlan("c = limit b 10;");
@@ -561,7 +570,7 @@
}
@Test
- public void testQueryDistinctWithSchema() {
+ public void testQueryDistinctWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url,hitCount);");
LogicalPlan lp = planTester.buildPlan("b = distinct a;");
@@ -582,7 +591,7 @@
}
@Test
- public void testQueryStreamingWithSchema() {
+ public void testQueryStreamingWithSchema() throws FrontendException {
String query = "stream (load 'a') through `" + simpleEchoStreamingCommand + "` as (x, y);";
LogicalPlan lp = planTester.buildPlan(query);
@@ -594,7 +603,7 @@
}
@Test
- public void testQueryStreamingWithSchema1() {
+ public void testQueryStreamingWithSchema1() throws FrontendException {
String query = "stream (load 'a' as (url, hitCount)) through `" + simpleEchoStreamingCommand + "` as (x, y);";
LogicalPlan lp = planTester.buildPlan(query);
@@ -606,7 +615,7 @@
}
@Test
- public void testQueryImplicitJoinWithSchema() {
+ public void testQueryImplicitJoinWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url,hitCount);");
planTester.buildPlan("b = load 'b' as (url,rank);");
planTester.buildPlan("c = cogroup a by url, b by url;");
@@ -683,7 +692,7 @@
}
@Test
- public void testQueryCrossWithSchema(){
+ public void testQueryCrossWithSchema() throws FrontendException {
String query = "c = cross (load 'a' as (a, b, c)), (load 'b' as (d, e, f));";
LogicalPlan lp = planTester.buildPlan(query);
@@ -740,7 +749,7 @@
}
@Test
- public void testQueryUnionWithSchema(){
+ public void testQueryUnionWithSchema() throws FrontendException {
String query = "c = union (load 'a' as (url, hitcount)), (load 'b' as (url, rank));";
LogicalPlan lp = planTester.buildPlan(query);
@@ -759,7 +768,7 @@
}
@Test
- public void testQueryFRJoinSchema(){
+ public void testQueryFRJoinSchema() throws FrontendException {
String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b' as (url, rank)) by $0 using \"replicated\";";
LogicalPlan lp = planTester.buildPlan(query);
@@ -804,7 +813,7 @@
}
@Test
- public void testQueryJoinWithSchema(){
+ public void testQueryJoinWithSchema() throws FrontendException {
String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b' as (url, rank)) by $0;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -849,29 +858,16 @@
}
@Test
- public void testQueryCrossWithMixedSchema(){
+ public void testQueryCrossWithMixedSchema() throws FrontendException {
String query = "c = cross (load 'a' as (url, hitcount)), (load 'b');";
LogicalPlan lp = planTester.buildPlan(query);
LOCross cross = (LOCross)lp.getLeaves().get(0);
- RequiredFields crossRelevantFields0 = cross.getRelevantInputs(0, 0).get(0);
- assertTrue(crossRelevantFields0.getNeedAllFields()==false);
- assertTrue(crossRelevantFields0.getNeedNoFields()==false);
- assertTrue(crossRelevantFields0.getFields().get(0).first==0);
- assertTrue(crossRelevantFields0.getFields().get(0).second==0);
-
- RequiredFields crossRelevantFields1 = cross.getRelevantInputs(0, 1).get(0);
- assertTrue(crossRelevantFields1.getNeedAllFields()==false);
- assertTrue(crossRelevantFields1.getNeedNoFields()==false);
- assertTrue(crossRelevantFields1.getFields().get(0).first==0);
- assertTrue(crossRelevantFields1.getFields().get(0).second==1);
-
- assertTrue(cross.getRelevantInputs(0, 2)==null);
+ assertTrue(cross.getRelevantInputs(0, 0)==null);
}
-
@Test
- public void testQueryUnionWithMixedSchema(){
+ public void testQueryUnionWithMixedSchema() throws FrontendException {
String query = "c = union (load 'a' as (url, hitcount)), (load 'b');";
LogicalPlan lp = planTester.buildPlan(query);
LOUnion union = (LOUnion)lp.getLeaves().get(0);
@@ -890,53 +886,25 @@
}
@Test
- public void testQueryFRJoinWithMixedSchema(){
+ public void testQueryFRJoinWithMixedSchema() throws FrontendException {
String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b') by $0 using \"replicated\";";
LogicalPlan lp = planTester.buildPlan(query);
LOJoin frjoin = (LOJoin)lp.getLeaves().get(0);
- RequiredFields frjoinRelevantFields0 = frjoin.getRelevantInputs(0, 0).get(0);
- assertTrue(frjoinRelevantFields0.getNeedAllFields() == false);
- assertTrue(frjoinRelevantFields0.getNeedNoFields() == false);
- assertTrue(frjoinRelevantFields0.getFields().size() == 1);
- assertTrue(frjoinRelevantFields0.getFields().get(0).first == 0);
- assertTrue(frjoinRelevantFields0.getFields().get(0).second == 0);
-
- RequiredFields frjoinRelevantFields1 = frjoin.getRelevantInputs(0, 1).get(0);
- assertTrue(frjoinRelevantFields1.getNeedAllFields() == false);
- assertTrue(frjoinRelevantFields1.getNeedNoFields() == false);
- assertTrue(frjoinRelevantFields1.getFields().size() == 1);
- assertTrue(frjoinRelevantFields1.getFields().get(0).first == 0);
- assertTrue(frjoinRelevantFields1.getFields().get(0).second == 1);
-
- assertTrue(frjoin.getRelevantInputs(0, 2)==null);
+ assertTrue(frjoin.getRelevantInputs(0, 0)==null);
}
@Test
- public void testQueryJoinWithMixedSchema(){
+ public void testQueryJoinWithMixedSchema() throws FrontendException {
String query = "c = join (load 'a' as (url, hitcount)) by $0, (load 'b') by $0;";
LogicalPlan lp = planTester.buildPlan(query);
LOJoin join = (LOJoin)lp.getLeaves().get(0);
- RequiredFields joinRelevantFields0 = join.getRelevantInputs(0, 0).get(0);
- assertTrue(joinRelevantFields0.getNeedAllFields() == false);
- assertTrue(joinRelevantFields0.getNeedNoFields() == false);
- assertTrue(joinRelevantFields0.getFields().size() == 1);
- assertTrue(joinRelevantFields0.getFields().get(0).first == 0);
- assertTrue(joinRelevantFields0.getFields().get(0).second == 0);
-
- RequiredFields joinRelevantFields1 = join.getRelevantInputs(0, 1).get(0);
- assertTrue(joinRelevantFields1.getNeedAllFields() == false);
- assertTrue(joinRelevantFields1.getNeedNoFields() == false);
- assertTrue(joinRelevantFields1.getFields().size() == 1);
- assertTrue(joinRelevantFields1.getFields().get(0).first == 0);
- assertTrue(joinRelevantFields1.getFields().get(0).second == 1);
-
- assertTrue(join.getRelevantInputs(0, 2)==null);
+ assertTrue(join.getRelevantInputs(0, 0)==null);
}
@Test
- public void testQueryFilterWithStarNoSchema() {
+ public void testQueryFilterWithStarNoSchema() throws FrontendException {
planTester.buildPlan("a = load 'a';");
LogicalPlan lp = planTester.buildPlan("b = filter a by COUNT(*) == 3;");
@@ -957,7 +925,7 @@
}
@Test
- public void testQueryOrderByStarNoSchema() {
+ public void testQueryOrderByStarNoSchema() throws FrontendException {
planTester.buildPlan("a = load 'a';");
LogicalPlan lp = planTester.buildPlan("b = order a by *;");
@@ -978,7 +946,7 @@
}
@Test
- public void testQueryGroupByStarNoSchema() throws Exception {
+ public void testQueryGroupByStarNoSchema() throws FrontendException {
String query = "foreach (group (load 'a') by *) generate $1 ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1003,7 +971,7 @@
}
@Test
- public void testQueryFRJoinOnStarNoSchema(){
+ public void testQueryFRJoinOnStarNoSchema() throws FrontendException {
String query = "c = join (load 'a') by *, (load 'b') by * using \"replicated\";";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1012,7 +980,7 @@
}
@Test
- public void testQueryJoinOnStarNoSchema(){
+ public void testQueryJoinOnStarNoSchema() throws FrontendException {
String query = "c = join (load 'a') by *, (load 'b') by *;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1021,7 +989,7 @@
}
@Test
- public void testQueryFilterStarWithSchema() {
+ public void testQueryFilterStarWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url,hitCount);");
LogicalPlan lp = planTester.buildPlan("b = filter a by COUNT(*) == 3;");
@@ -1042,15 +1010,24 @@
}
@Test
- public void testQuerySplitWithStarSchema() {
+ public void testQuerySplitWithStarSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url, hitCount);");
LogicalPlan lp = planTester.buildPlan("split a into b if url == '3', c if COUNT(*) == '3';");
LOSplit split = (LOSplit)lp.getSuccessors(lp.getRoots().get(0)).get(0);
- RequiredFields splitRelevantFields = split.getRelevantInputs(0, 0).get(0);
- assertTrue(splitRelevantFields.needAllFields() == false);
- assertTrue(splitRelevantFields.needNoFields() == true);
- assertTrue(splitRelevantFields.getFields() == null);
+ RequiredFields splitRelevantFields0 = split.getRelevantInputs(0, 0).get(0);
+ assertTrue(splitRelevantFields0.needAllFields() == false);
+ assertTrue(splitRelevantFields0.needNoFields() == false);
+ assertTrue(splitRelevantFields0.getFields().size() == 1);
+ assertTrue(splitRelevantFields0.getFields().contains(new Pair<Integer, Integer>(0, 0)));
+
+ RequiredFields splitRelevantFields1 = split.getRelevantInputs(0, 1).get(0);
+ assertTrue(splitRelevantFields1.needAllFields() == false);
+ assertTrue(splitRelevantFields1.needNoFields() == false);
+ assertTrue(splitRelevantFields1.getFields().size() == 1);
+ assertTrue(splitRelevantFields1.getFields().contains(new Pair<Integer, Integer>(0, 1)));
+
+ assertTrue(split.getRelevantInputs(0, 2) == null);
LOSplitOutput splitb = (LOSplitOutput)lp.getSuccessors(split).get(0);
RequiredFields splitbRelevantFields0 = splitb.getRelevantInputs(0, 0).get(0);
@@ -1081,7 +1058,7 @@
}
@Test
- public void testQueryOrderByStarWithSchema() {
+ public void testQueryOrderByStarWithSchema() throws FrontendException {
planTester.buildPlan("a = load 'a' as (url,hitCount);");
LogicalPlan lp = planTester.buildPlan("b = order a by *;");
@@ -1100,7 +1077,7 @@
}
@Test
- public void testQueryGroupByStarWithSchema() throws Exception {
+ public void testQueryGroupByStarWithSchema() throws FrontendException {
String query = "foreach (group (load 'a' as (url, hitCount)) by *) generate $1 ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1131,7 +1108,7 @@
}
@Test
- public void testQueryFRJoinOnStarWithSchema(){
+ public void testQueryFRJoinOnStarWithSchema() throws FrontendException {
String query = "c = join (load 'a' as (url, hitcount)) by *, (load 'b' as (url, rank)) by * using \"replicated\";";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1176,7 +1153,7 @@
}
@Test
- public void testQueryJoinOnStarWithSchema(){
+ public void testQueryJoinOnStarWithSchema() throws FrontendException {
String query = "c = join (load 'a' as (url, hitcount)) by *, (load 'b' as (url, rank)) by *;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1221,7 +1198,7 @@
}
@Test
- public void testQueryForeachGenerateStarNoSchema() {
+ public void testQueryForeachGenerateStarNoSchema() throws FrontendException {
String query = "foreach (load 'a') generate * ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1230,7 +1207,7 @@
}
@Test
- public void testQueryForeachGenerateCountStarNoSchema() {
+ public void testQueryForeachGenerateCountStarNoSchema() throws FrontendException {
String query = "foreach (load 'a') generate COUNT(*) ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1242,7 +1219,7 @@
}
@Test
- public void testQueryForeachGenerateStarNoSchema1() {
+ public void testQueryForeachGenerateStarNoSchema1() throws FrontendException {
String query = "foreach (load 'a') generate *, COUNT(*) ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1252,7 +1229,7 @@
}
@Test
- public void testQueryForeachGenerateStarNoSchema2() {
+ public void testQueryForeachGenerateStarNoSchema2() throws FrontendException {
String query = "foreach (load 'a') generate *, $0 ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1262,7 +1239,7 @@
}
@Test
- public void testQueryForeachGenerateStarWithSchema() {
+ public void testQueryForeachGenerateStarWithSchema() throws FrontendException {
String query = "foreach (load 'a' as (url, hitCount)) generate * ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1285,7 +1262,7 @@
}
@Test
- public void testQueryForeachGenerateCountStarWithSchema() {
+ public void testQueryForeachGenerateCountStarWithSchema() throws FrontendException {
String query = "foreach (load 'a' as (url, hitCount)) generate COUNT(*) ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1297,7 +1274,7 @@
}
@Test
- public void testQueryForeachGenerateStarWithSchema1() {
+ public void testQueryForeachGenerateStarWithSchema1() throws FrontendException {
String query = "foreach (load 'a' as (url, hitCount)) generate *, COUNT(*) ;";
LogicalPlan lp = planTester.buildPlan(query);
@@ -1323,7 +1300,7 @@
}
@Test
- public void testQueryForeachGenerateStarWithSchema2() {
+ public void testQueryForeachGenerateStarWithSchema2() throws FrontendException {
String query = "foreach (load 'a' as (url, hitCount)) generate *, url ;";
LogicalPlan lp = planTester.buildPlan(query);
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSkewedJoin.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSkewedJoin.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSkewedJoin.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestSkewedJoin.java Thu Dec 17 18:36:28 2009
@@ -42,6 +42,8 @@
private static final String INPUT_FILE3 = "SkewedJoinInput3.txt";
private static final String INPUT_FILE4 = "SkewedJoinInput4.txt";
private static final String INPUT_FILE5 = "SkewedJoinInput5.txt";
+ private static final String INPUT_FILE6 = "SkewedJoinInput6.txt";
+ private static final String INPUT_FILE7 = "SkewedJoinInput7.txt";
private PigServer pigServer;
private MiniCluster cluster = MiniCluster.buildCluster();
@@ -114,12 +116,31 @@
w5.println("100\t");
w5.close();
+ PrintWriter w6 = new PrintWriter(new FileWriter(INPUT_FILE6));
+
+ for(int i=0; i<300; i++) {
+ for(int j=0; j<5; j++) {
+ w6.println(""+i+"\t"+j);
+ }
+ }
+ w6.close();
+
+ PrintWriter w7 = new PrintWriter(new FileWriter(INPUT_FILE7));
+
+ for(int i=0; i<300; i = i+3) {
+ for(int j=0; j<2; j++) {
+ w7.println(""+i+"\t"+j);
+ }
+ }
+ w7.close();
+
Util.copyFromLocalToCluster(cluster, INPUT_FILE1, INPUT_FILE1);
Util.copyFromLocalToCluster(cluster, INPUT_FILE2, INPUT_FILE2);
Util.copyFromLocalToCluster(cluster, INPUT_FILE3, INPUT_FILE3);
Util.copyFromLocalToCluster(cluster, INPUT_FILE4, INPUT_FILE4);
Util.copyFromLocalToCluster(cluster, INPUT_FILE5, INPUT_FILE5);
-
+ Util.copyFromLocalToCluster(cluster, INPUT_FILE6, INPUT_FILE6);
+ Util.copyFromLocalToCluster(cluster, INPUT_FILE7, INPUT_FILE7);
}
@After
@@ -128,6 +149,9 @@
new File(INPUT_FILE2).delete();
new File(INPUT_FILE3).delete();
new File(INPUT_FILE4).delete();
+ new File(INPUT_FILE5).delete();
+ new File(INPUT_FILE6).delete();
+ new File(INPUT_FILE7).delete();
Util.deleteDirectory(new File("skewedjoin"));
Util.deleteFile(cluster, INPUT_FILE1);
@@ -135,7 +159,8 @@
Util.deleteFile(cluster, INPUT_FILE3);
Util.deleteFile(cluster, INPUT_FILE4);
Util.deleteFile(cluster, INPUT_FILE5);
-
+ Util.deleteFile(cluster, INPUT_FILE6);
+ Util.deleteFile(cluster, INPUT_FILE7);
}
public void testSkewedJoinWithGroup() throws IOException{
@@ -392,4 +417,31 @@
Assert.assertEquals(true, TestHelper.compareBags(dbfrj, dbrj));
}
+
+ public void testSkewedJoinManyReducers() throws IOException {
+ pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.maxtuple", "2");
+ pigServer.registerQuery("A = LOAD '" + INPUT_FILE6 + "' as (id,name);");
+ pigServer.registerQuery("B = LOAD '" + INPUT_FILE7 + "' as (id,name);");
+
+ DataBag dbfrj = BagFactory.getInstance().newDefaultBag(), dbrj = BagFactory.getInstance().newDefaultBag();
+ {
+ pigServer.registerQuery("E = join A by id, B by id using \"skewed\" parallel 300;");
+ Iterator<Tuple> iter = pigServer.openIterator("E");
+
+ while(iter.hasNext()) {
+ dbfrj.add(iter.next());
+ }
+ }
+ {
+ pigServer.registerQuery("E = join A by id, B by id;");
+ Iterator<Tuple> iter = pigServer.openIterator("E");
+
+ while(iter.hasNext()) {
+ dbrj.add(iter.next());
+ }
+ }
+ Assert.assertEquals(dbfrj.size(), dbrj.size());
+ Assert.assertEquals(true, TestHelper.compareBags(dbfrj, dbrj));
+
+ }
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeChecking.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeChecking.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeChecking.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeChecking.java Thu Dec 17 18:36:28 2009
@@ -18,8 +18,11 @@
package org.apache.pig.test;
+import junit.framework.Assert;
import junit.framework.TestCase;
import org.apache.pig.impl.logicalLayer.*;
+import org.apache.pig.impl.logicalLayer.validators.TypeCheckerException;
+import org.apache.pig.impl.plan.PlanValidationException;
import org.apache.pig.test.utils.TypeCheckingTestUtil;
import org.apache.pig.test.utils.LogicalPlanTester;
import org.junit.Test;
@@ -247,7 +250,18 @@
planTester.buildPlan("a = load '/user/pig/tests/data/singlefile/studenttab10k' as (name:chararray, age:int, gpa:double);") ;
LogicalPlan plan1 = planTester.buildPlan("b = foreach a generate (long)age as age:long, (int)gpa as gpa:int;") ;
LogicalPlan plan2 = planTester.buildPlan("c = foreach b generate SUM(age), SUM(gpa);") ;
- planTester.typeCheckPlan(plan2);
+ try {
+ planTester.typeCheckPlan(plan2);
+ } catch (PlanValidationException e) {
+ Throwable t = e.getCause();
+ if (!(t instanceof TypeCheckerException)) {
+ System.out.println("t: " + t);
+ Assert.fail();
+ }
+ return;
+ }
+ // shouldn't get here
+ Assert.fail();
}
public void testSUM2() throws Throwable {
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestTypeCheckingValidatorNoSchema.java Thu Dec 17 18:36:28 2009
@@ -28,6 +28,7 @@
import org.apache.pig.impl.plan.CompilationMessageCollector;
import org.apache.pig.impl.plan.PlanValidationException;
import org.apache.pig.impl.util.MultiMap;
+import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema ;
@@ -36,6 +37,7 @@
import java.util.List;
import java.util.ArrayList;
+import java.util.Properties;
public class TestTypeCheckingValidatorNoSchema extends TestCase {
@@ -443,7 +445,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, new HDataStorage(new Properties()), true) ;
// set schemas
load1.setEnforcedSchema(null) ;
@@ -507,7 +509,7 @@
LOLoad load1 = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, new HDataStorage(new Properties()), true) ;
// set schemas
load1.setEnforcedSchema(null) ;
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/Util.java Thu Dec 17 18:36:28 2009
@@ -396,23 +396,8 @@
String replacement = quoteReplacement("\\\\");
return str.replaceAll(regex, replacement);
}
-
- /**
- * Helper method to construct URI for local file system. For unix, it will
- * put "file:" in the front of the path; For Windows, it will put "file:/" in
- * front of the path, and also call encodeEscape to replace "\" with "\\"
- *
- * @param str absolute path (under cygwin, should be a windows style path)
- * @return The resulting string
- */
- public static String generateURI(String path)
- {
- if (System.getProperty("os.name").toUpperCase().startsWith("WINDOWS"))
- return "file:/"+encodeEscape(path);
- return "file:"+path;
- }
- static String generateURI(String filename, PigContext context)
+ public static String generateURI(String filename, PigContext context)
throws IOException {
if (context.getExecType() == ExecType.MAPREDUCE) {
return FileLocalizer.hadoopify(filename, context);
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan10.dot Thu Dec 17 18:36:28 2009
@@ -1,7 +1,7 @@
digraph graph1 {
- LOLoad1061 -> LOLimit1062;
- LOLimit1062 [limit="100"];
- LOLimit1062 -> LOForEach1067;
- LOForEach1067 [hasflat="false"];
- LOForEach1067 -> LOCogroup1064;
+ LOLoad2061 -> LOLimit2062;
+ LOLimit2062 [limit="100"];
+ LOLimit2062 -> LOForEach2067;
+ LOForEach2067 [hasflat="false"];
+ LOForEach2067 -> LOCogroup2064;
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan4.dot Thu Dec 17 18:36:28 2009
@@ -1,7 +1,7 @@
digraph graph1 {
- LOLoad1029 -> LOCogroup1031;
- LOCogroup1031 -> LOForEach1035;
- LOForEach1035 [hasflat="true"];
- LOForEach1035 -> LOLimit1036;
- LOLimit1036 [limit="100"];
+ LOLoad2029 -> LOCogroup2031;
+ LOCogroup2031 -> LOForEach2035;
+ LOForEach2035 [hasflat="true"];
+ LOForEach2035 -> LOLimit2036;
+ LOLimit2036 [limit="100"];
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan5.dot Thu Dec 17 18:36:28 2009
@@ -1,6 +1,6 @@
digraph graph1 {
- LOLoad1037 -> LOLimit1042;
- LOLimit1042 [limit="100"];
- LOLimit1042 -> LOForEach1041;
- LOForEach1041 [hasflat="false"];
+ LOLoad2037 -> LOLimit2042;
+ LOLimit2042 [limit="100"];
+ LOLimit2042 -> LOForEach2041;
+ LOForEach2041 [hasflat="false"];
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan6.dot Thu Dec 17 18:36:28 2009
@@ -1,4 +1,4 @@
digraph graph1 {
- LOLoad1043 -> LOLimit1044;
- LOLimit1044 [limit="20"];
+ LOLoad2043 -> LOLimit2044;
+ LOLimit2044 [limit="20"];
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan7.dot Thu Dec 17 18:36:28 2009
@@ -1,6 +1,6 @@
digraph graph1 {
- LOLoad1047 -> LOForEach1051;
- LOForEach1051 [hasflat="true"];
- LOForEach1051 -> LOLimit1052;
- LOLimit1052 [limit="100"];
+ LOLoad2047 -> LOForEach2051;
+ LOForEach2051 [hasflat="true"];
+ LOForEach2051 -> LOLimit2052;
+ LOLimit2052 [limit="100"];
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan8.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan8.dot?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan8.dot (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan8.dot Thu Dec 17 18:36:28 2009
@@ -1,6 +1,6 @@
digraph graph1 {
- LOLoad1053 -> LOSort1055;
- LOSort1055 [limit="-1"];
- LOSort1055 -> LOLimit1056;
- LOLimit1056 [limit="10"];
+ LOLoad2053 -> LOSort2055;
+ LOSort2055 [limit="-1"];
+ LOSort2055 -> LOLimit2056;
+ LOLimit2056 [limit="10"];
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan9.dot
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan9.dot?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan9.dot (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/DotFiles/optlimitplan9.dot Thu Dec 17 18:36:28 2009
@@ -1,4 +1,4 @@
digraph graph1 {
- LOLoad1057 -> LOSort1059;
- LOSort1059 [limit="10"];
+ LOLoad2057 -> LOSort2059;
+ LOSort2059 [limit="10"];
}
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult.pig Thu Dec 17 18:36:28 2009
@@ -8,4 +8,4 @@
store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
grpInactiveAcct = group inactiveAccounts all;
countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
-store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult2.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult2.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult2.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult2.pig Thu Dec 17 18:36:28 2009
@@ -8,4 +8,4 @@
store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
grpInactiveAcct = group inactiveAccounts all;
countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
-store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
\ No newline at end of file
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult4.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult4.pig?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult4.pig (added)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResult4.pig Thu Dec 17 18:36:28 2009
@@ -0,0 +1,14 @@
+
+
+
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig (added)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultCmdLnPriorDeclare.pig Thu Dec 17 18:36:28 2009
@@ -0,0 +1,12 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultComment.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultComment.pig?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultComment.pig (added)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultComment.pig Thu Dec 17 18:36:28 2009
@@ -0,0 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
+
+/* comment */
+
+/* l = load '$zxv';
+ store l into '$zxcv';
+*/
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate /* $col 1 */ $0,$12,$7;
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- testing $comment
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultDefault.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultDefault.pig?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultDefault.pig (added)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultDefault.pig Thu Dec 17 18:36:28 2009
@@ -0,0 +1,12 @@
+
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig (added)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/ExpectedResultMulDecs.pig Thu Dec 17 18:36:28 2009
@@ -0,0 +1,13 @@
+
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld Thu Dec 17 18:36:28 2009
@@ -20,4 +20,4 @@
| | |
| | Constant(all) - scope-127
| |
- | |---Load(file:/tmp/input2:org.apache.pig.impl.builtin.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaabfkaaangfgogeepggebgmgmejgohahfhehihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgogjbebnmkonpeflhcacaaagemaaakgneghcgpgnefgeghgfhdheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlemaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaignfegpefgeghgfhdhbaahoaaaehihahdhcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbh
ggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahiheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaaffkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghfkaaaehdhegbhcemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhedlhihcaagcgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhd
gjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaalfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpg
ogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaacecaaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdhfgmheheaabcemgkgbhggbcpgmgbgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaab
ejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahi','')) - scope-118
\ No newline at end of file
+ | |---Load(file:/tmp/input2:org.apache.pig.impl.builtin.MergeJoinIndexer('org.apache.pig.builtin.PigStorage','kmonaaafhdhcaabdgkgbhggbcohfhegjgmcoebhchcgbhjemgjhdhehiibncbnjjmhgbjnadaaabejaaaehdgjhkgfhihaaaaaaaabhhaeaaaaaaabhdhcaaeogphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccohagmgbgohdcofagihjhdgjgdgbgmfagmgbgoaaaaaaaaaaaaaaabacaaabfkaaangfgogeepggebgmgmejgohahfhehihcaacfgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcfagmgbgohlinhnlnngaiiihoacaaagemaaakgneghcgpgnefgeghgfhdheaacdemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphfhegjgmcpenhfgmhegjengbhadlemaaafgnelgfhjhdheaaapemgkgbhggbcphfhegjgmcpengbhadlemaaahgnemgfgbhggfhdheaabaemgkgbhggbcphfhegjgmcpemgjhdhedlemaaaegnephahdhbaahoaaafemaaaggnfcgpgphehdhbaahoaaagemaaaignfegpefgeghgfhdhbaahoaaaehihahdhcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohfhegjgmcoenhfgmhegjengbhaaaaaaaaaaaaaaaacacaaabemaaaegnengbhahbaahoaaafhihahdhcaabbgkgbh
ggbcohfhegjgmcoeigbhdgiengbhaafahnkmbmdbgganbadaaacegaaakgmgpgbgeeggbgdhegphcejaaajhegihcgfhdgigpgmgehihadpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhdhcaacegphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcelgfhjaaaaaaaaaaaaaaabacaaacekaaacgjgeemaaafhdgdgphagfheaabcemgkgbhggbcpgmgbgoghcpfdhehcgjgoghdlhihaaaaaaaaaaaaaaahiheaaafhdgdgphagfhdhcaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdheaaaaaaaaaaaaaaabacaaaffkaaakgphggfhcgmgpgbgegfgefkaabfhahcgpgdgfhdhdgjgoghecgbghepggfehfhagmgfhdfkaabehcgfhdhfgmhefdgjgoghgmgffehfhagmgfecgbghfkaaaehdhegbhcemaaahgdgpgmhfgngohdheaabfemgkgbhggbcphfhegjgmcpebhchcgbhjemgjhdhedlhihcaagcgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcoefhihahcgfhdhd
gjgpgoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaadgmgpghheaacaemgphcghcpgbhagbgdgigfcpgdgpgngngpgohdcpgmgpghghgjgoghcpemgpghdlhihcaaemgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofagihjhdgjgdgbgmephagfhcgbhegphcaaaaaaaaaaaaaaabacaaalfkaaafgbgdgdhfgnfkaaangjgohahfheebhehegbgdgigfgeejaabehcgfhbhfgfhdhegfgefagbhcgbgmgmgfgmgjhdgnecaaakhcgfhdhfgmhefehjhagfemaaafgjgohahfheheaablemgphcghcpgbhagbgdgigfcphagjghcpgegbhegbcpfehfhagmgfdlemaaaggjgohahfhehdhbaahoaaagemaaangmgjgogfgbghgffehcgbgdgfhcheaachemgphcghcpgbhagbgdgigfcphagjghcphagfgocphfhegjgmcpemgjgogfgbghgffehcgbgdgfhcdlemaaadgmgpghhbaahoaabeemaaahgphfhehahfhehdhbaahoaaagemaaakhagbhcgfgohefagmgbgoheaafaemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpgogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccphagmgbgohdcpfagihjhdgjgdgbgmfagmgbgodlemaaadhcgfhdheaaeeemgphcghcpgbhagbgdgigfcphagjghcpgcgbgdglgfgogecpgigbgegpgphacpgfhigfgdhfhegjgpg
ogfgoghgjgogfcphagihjhdgjgdgbgmemgbhjgfhccpfcgfhdhfgmhedlhihcaacbgphcghcogbhagbgdgigfcohagjghcogjgnhagmcohagmgbgocoephagfhcgbhegphcaaaaaaaaaaaaaaabacaaabemaaaegnelgfhjheaacgemgphcghcpgbhagbgdgigfcphagjghcpgjgnhagmcphagmgbgocpephagfhcgbhegphcelgfhjdlhihahbaahoaaapaaaappppppppdchahahahdhcaaclgphcghcogbhagbgdgigfcogdgpgngngpgohdcogmgpghghgjgoghcogjgnhagmcoemgpghdeekemgpghghgfhccikmpnoicknfncdiacaaabemaaaegogbgngfhbaahoaaaohihaheaafjgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccogfhihahcgfhdhdgjgpgoephagfhcgbhegphchdcofaepfahcgpgkgfgdhehahahdhcaaecgphcghcogbhagbgdgigfcohagjghcogcgbgdglgfgogecogigbgegpgphacogfhigfgdhfhegjgpgogfgoghgjgogfcohagihjhdgjgdgbgmemgbhjgfhccofcgfhdhfgmheaaaaaaaaaaaaaaabacaaacecaaamhcgfhehfhcgofdhegbhehfhdemaaaghcgfhdhfgmheheaabcemgkgbhggbcpgmgbgoghcpepgcgkgfgdhedlhihaachahbaahoaaboaaaaaaaahdhbaahoaaaaaaaaaaabhhaeaaaaaaabhdhcaabbgkgbhggbcogmgbgoghcoejgohegfghgfhcbcockakephibihdiacaaab
ejaaafhggbgmhfgfhihcaabagkgbhggbcogmgbgoghcoeohfgngcgfhcigkmjfbnaljeoailacaaaahihaaaaaaaaahihihdhbaahoaaaaaaaaaaabhhaeaaaaaaakhbaahoaabmhihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaabhbaahoaabmhbaahoaaaphihdhbaahoaaaaaaaaaaaahhaeaaaaaaakhihdhbaahoaaaihdhbaahoaaakdpeaaaaaaaaaaaamhhaiaaaaaabaaaaaaaaahiaahi','')) - scope-118
\ No newline at end of file
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/input5.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/input5.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/input5.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/input5.pig Thu Dec 17 18:36:28 2009
@@ -1,5 +1,4 @@
%declare udfs /data/pigudf.jar
-
register $udfs;
aa = load '$loadfile ' using PigStorage('\x01');
Added: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputComment.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputComment.pig?rev=891828&view=auto
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputComment.pig (added)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputComment.pig Thu Dec 17 18:36:28 2009
@@ -0,0 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
+%default date /* comment */ 1000 -- setting default
+/* comment */ %declare arity 16
+
+/* l = load '$zxv';
+ store l into '$zxcv';
+*/
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
+bb = filter aa by (ARITY == '$arity') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
+a = foreach bb generate /* $col 1 */ $0,$12,$7;
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- testing $comment
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputDefault.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputDefault.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputDefault.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputDefault.pig Thu Dec 17 18:36:28 2009
@@ -1,5 +1,4 @@
%default date '20080228'
-
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
a = foreach bb generate $0,$12,$7;
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputEscape.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputEscape.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputEscape.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputEscape.pig Thu Dec 17 18:36:28 2009
@@ -1,5 +1,4 @@
%default column1 'asdf'
-
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
a = foreach bb generate $0,$12,$7;
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputSubstitutionWithinShellCommand.pig Thu Dec 17 18:36:28 2009
@@ -1,4 +1,5 @@
+
%DECLARE date '20080228'
%DECLARE formatted_date `sh test/org/apache/pig/test/data/generate_date.sh $date`
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$formatted_date' using PigStorage('\x01');
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputThreeParams.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputThreeParams.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputThreeParams.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/inputThreeParams.pig Thu Dec 17 18:36:28 2009
@@ -1,9 +1,7 @@
%default tableName 'value to be skipped'
-
aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
a = foreach bb generate $0,$12,$7;
-
%declare tableName 'inactiveAccounts'
--generate inactive accts
$tableName = filter a by ($1 neq '') and ($1 == '2') parallel 400;
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/output1.pig
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/output1.pig?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/output1.pig (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/data/output1.pig Thu Dec 17 18:36:28 2009
@@ -1,7 +1,18 @@
+-- %declare date '20080228'
+-- l = load '$xyz';
-register /data/pigudf.jar;
+/* comment */
-aa = load '/user/pig/tests/data/singlefile/textdoc.txt ' using PigStorage('\x01');
+/* l = load '$zxv';
+ store l into '$zxcv';
+*/
+aa = load '/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228' using PigStorage('\x01');
bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq 'ss') parallel 400;
-a = foreach bb generate $0,$12,$7;
-store inactiveAccounts into '/user/pig/tests/results/DefineClause_4.out';
+a = foreach bb generate /* $col 1 */ $0,$12,$7;
+/* $date /* */
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400; -- testing $comment
+store inactiveAccounts into '/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts ); }
+store countInactiveAcct into '/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java?rev=891828&r1=891827&r2=891828&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/utils/TypeCheckingTestUtil.java Thu Dec 17 18:36:28 2009
@@ -18,6 +18,9 @@
package org.apache.pig.test.utils;
+import org.apache.hadoop.hdfs.server.common.HdfsConstants.NodeType;
+import org.apache.pig.backend.datastorage.DataStorage;
+import org.apache.pig.backend.hadoop.datastorage.HDataStorage;
import org.apache.pig.FuncSpec;
import org.apache.pig.impl.logicalLayer.LogicalPlan;
import org.apache.pig.impl.logicalLayer.LOLoad;
@@ -32,17 +35,19 @@
import java.util.Iterator;
import java.util.List;
import java.util.ArrayList;
+import java.util.Properties;
import java.io.IOException;
public class TypeCheckingTestUtil {
public static LOLoad genDummyLOLoad(LogicalPlan plan) {
String pigStorage = PigStorage.class.getName() ;
+ DataStorage dfs = new HDataStorage(new Properties());
try {
LOLoad load = new LOLoad(plan,
genNewOperatorKey(),
new FileSpec("pi", new FuncSpec(pigStorage)),
- null, null, true) ;
+ null, dfs, true) ;
return load ;
} catch (IOException e) {
throw new AssertionError("This cannot happen") ;