You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2012/08/01 23:40:34 UTC
svn commit: r1368271 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/parser/QueryParserDriver.java
src/org/apache/pig/parser/QueryParserUtils.java
test/org/apache/pig/test/TestMacroExpansion.java
Author: daijy
Date: Wed Aug 1 21:40:33 2012
New Revision: 1368271
URL: http://svn.apache.org/viewvc?rev=1368271&view=rev
Log:
PIG-2729: Macro expansion does not use pig.import.search.path - UnitTest borked
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Aug 1 21:40:33 2012
@@ -475,6 +475,8 @@ PIG-2228: support partial aggregation in
BUG FIXES
+PIG-2729: Macro expansion does not use pig.import.search.path - UnitTest borked (johannesch via daijy)
+
PIG-2783: Fix Iterator_1 e2e test for Hadoop 23 (rohini via daijy)
PIG-2761: With hadoop23 importing modules inside python script does not work (rohini via daijy)
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Wed Aug 1 21:40:33 2012
@@ -19,6 +19,8 @@
package org.apache.pig.parser;
import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
@@ -344,10 +346,13 @@ public class QueryParserDriver {
private FetchFileRet getMacroFile(String fname) {
FetchFileRet localFileRet = null;
try {
- if (fnameMap.get(fname)!=null) {
+ if (fnameMap.get(fname) != null) {
localFileRet = fnameMap.get(fname);
} else {
- localFileRet = FileLocalizer.fetchFile(pigContext.getProperties(), fname);
+ File localFile = QueryParserUtils.getFileFromImportSearchPath(fname);
+ localFileRet = localFile == null ?
+ FileLocalizer.fetchFile(pigContext.getProperties(), fname)
+ : new FetchFileRet(localFile.getCanonicalFile(), false);
fnameMap.put(fname, localFileRet);
}
} catch (IOException e) {
@@ -443,7 +448,7 @@ public class QueryParserDriver {
BufferedReader in = null;
try {
- in = QueryParserUtils.getImportScriptAsReader(localFileRet.file.getAbsolutePath());
+ in = new BufferedReader(new FileReader(localFileRet.file));
} catch (FileNotFoundException e) {
String msg = getErrorMessage(fname, t,
"Failed to import file '" + fname + "'", e.getMessage());
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java Wed Aug 1 21:40:33 2012
@@ -18,10 +18,7 @@
package org.apache.pig.parser;
-import java.io.BufferedReader;
import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileReader;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -54,7 +51,7 @@ import org.apache.pig.tools.pigstats.Scr
public class QueryParserUtils {
private static Log log = LogFactory.getLog( LogicalPlanGenerator.class );
- public static String removeQuotes(String str) {
+ public static String removeQuotes(String str) {
if (str.startsWith("\u005c'") && str.endsWith("\u005c'"))
return str.substring(1, str.length() - 1);
else
@@ -201,12 +198,11 @@ public class QueryParserUtils {
}
}
- static BufferedReader getImportScriptAsReader(String scriptPath)
- throws FileNotFoundException {
+ static File getFileFromImportSearchPath(String scriptPath) {
File f = new File(scriptPath);
if (f.exists() || f.isAbsolute() || scriptPath.startsWith("./")
|| scriptPath.startsWith("../")) {
- return new BufferedReader(new FileReader(f));
+ return f;
}
ScriptState state = ScriptState.get();
@@ -218,14 +214,13 @@ public class QueryParserUtils {
for (String path : paths) {
File f1 = new File(path + File.separator + scriptPath);
if (f1.exists()) {
- return new BufferedReader(new FileReader(f1));
+ return f1;
}
}
}
}
- throw new FileNotFoundException("Can't find the Specified file "
- + scriptPath);
+ return null;
}
static QueryParser createParser(CommonTokenStream tokens) {
Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1368271&r1=1368270&r2=1368271&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Wed Aug 1 21:40:33 2012
@@ -19,6 +19,7 @@
package org.apache.pig.test;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import java.io.BufferedReader;
import java.io.File;
@@ -44,6 +45,10 @@ import org.junit.Test;
public class TestMacroExpansion {
+ private static String groupAndCountMacro = "define group_and_count (A,group_key, reducers) returns B {\n" +
+ " $B = distinct $A partition BY org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
+ "};\n";
+ private static String garbageMacroContent = "&?:#garbage-!";
static File command;
@BeforeClass
@@ -102,10 +107,6 @@ public class TestMacroExpansion {
@Test
public void distinctTest() throws Exception {
- String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
- " $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
- "};\n";
-
String script =
"alpha = load 'users' as (user, age, zip);\n" +
"gamma = group_and_count (alpha, user, 23);\n" +
@@ -120,7 +121,7 @@ public class TestMacroExpansion {
"store gamma INTO 'byuser';\n" +
"store delta INTO 'byage';\n";
- verify(macro + script, expected);
+ verify(groupAndCountMacro + script, expected);
}
@Test
@@ -658,10 +659,6 @@ public class TestMacroExpansion {
@Test
public void duplicationTest() throws Throwable {
- String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
- " $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
- "};\n";
-
String script =
"alpha = load 'users' as (user, age, zip);\n" +
"gamma = group_and_count (alpha, user, 23);\n" +
@@ -671,21 +668,12 @@ public class TestMacroExpansion {
String expectedErr = "Reason: Duplicated macro name 'group_and_count'";
- validateFailure(macro + macro + script, expectedErr);
+ validateFailure(groupAndCountMacro + groupAndCountMacro + script, expectedErr);
}
@Test
public void simpleImportTest() throws Exception {
- String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
- " $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
- "};\n";
-
- File f = new File("mytest.pig");
- f.deleteOnExit();
-
- FileWriter fw = new FileWriter(f);
- fw.append(macro);
- fw.close();
+ createFile("mytest.pig", groupAndCountMacro);
String script =
"import 'mytest.pig';\n" +
@@ -703,66 +691,55 @@ public class TestMacroExpansion {
@Test
public void importUsingSearchPathTest() throws Exception {
- String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
- " $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
- "};\n";
-
- File f = new File("/tmp/mytest2.pig");
- f.deleteOnExit();
-
- FileWriter fw = new FileWriter(f);
- fw.append(macro);
- fw.close();
-
- String script =
- "import '/tmp/mytest2.pig';\n" +
- "alpha = load 'users' as (user, age, zip);\n" +
- "gamma = group_and_count (alpha, user, 23);\n" +
- "store gamma into 'byuser';\n";
-
- File f1 = new File("myscript.pig");
- f1.deleteOnExit();
-
- FileWriter fw1 = new FileWriter(f1);
- fw1.append(script);
- fw1.close();
-
- String[] args = { "-Dpig.import.search.path=/tmp", "-x", "local", "-c", "myscript.pig" };
- PigStats stats = PigRunner.run(args, null);
-
- assertTrue(stats.isSuccessful());
+ createFile("/tmp/mytest2.pig", garbageMacroContent);
+ assertTrue(verifyImportUsingSearchPath("/tmp/mytest2.pig", "mytest2.pig", "/tmp"));
+ }
+
+ @Test
+ public void importUsingSearchPathTest2() throws Exception {
+ createFile("/tmp/mytest2.pig", garbageMacroContent);
+ assertTrue(verifyImportUsingSearchPath("mytest2.pig", "./mytest2.pig", "/tmp"));
+ }
+
+ @Test
+ public void importUsingSearchPathTest3() throws Exception {
+ createFile("/tmp/mytest2.pig", garbageMacroContent);
+ assertTrue(verifyImportUsingSearchPath("../mytest2.pig", "../mytest2.pig", "/tmp"));
+ }
+
+ @Test
+ public void importUsingSearchPathTest4() throws Exception {
+ createFile("/tmp/pigtestdir/tmp/mytest2.pig", garbageMacroContent);
+ assertTrue(verifyImportUsingSearchPath("/tmp/mytest2.pig", "/tmp/mytest2.pig", "/tmp/pigtestdir/"));
+ }
+
+ @Test
+ public void importUsingSearchPathTest5() throws Exception {
+ createFile("/tmp/mytest2a.pig", garbageMacroContent);
+ createFile("mytest2a.pig", groupAndCountMacro);
+ assertTrue(verifyImportUsingSearchPath("/tmp/mytest2a.pig", "mytest2a.pig", "/tmp", false));
}
@Test
- public void negtiveUsingSearchPathTest() throws Exception {
- String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
- " $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
- "};\n";
-
- File f = new File("/tmp/mytest2.pig");
- f.deleteOnExit();
-
- FileWriter fw = new FileWriter(f);
- fw.append(macro);
- fw.close();
-
- String script =
- "import 'mytest2.pig';\n" +
- "alpha = load 'users' as (user, age, zip);\n" +
- "gamma = group_and_count (alpha, user, 23);\n" +
- "store gamma into 'byuser';\n";
-
- File f1 = new File("myscript.pig");
- f1.deleteOnExit();
-
- FileWriter fw1 = new FileWriter(f1);
- fw1.append(script);
- fw1.close();
-
- String[] args = { "-x", "local", "-c", "myscript.pig" };
- PigStats stats = PigRunner.run(args, null);
-
- assertTrue(!stats.isSuccessful());
+ public void negativeUsingSearchPathTest() throws Exception {
+ /* Delete the mytest3.pig file in negativeUsingSearchPathTest, just in
+ * case negativeUsingSearchPathTest2 is executed first and a garbage mytest3.pig
+ * file is created.
+ */
+ new File("mytest3.pig").delete();
+ assertFalse(verifyImportUsingSearchPath("/tmp/mytest3.pig", "mytest3.pig", null));
+ }
+
+ @Test
+ public void negativeUsingSearchPathTest2() throws Exception {
+ createFile("mytest3.pig", garbageMacroContent);
+ assertFalse(verifyImportUsingSearchPath("/tmp/mytest3.pig", "mytest3.pig", "/tmp"));
+ }
+
+ @Test
+ public void negativeUsingSearchPathTest3() throws Exception {
+ createFile("/tmp/mytest3a.pig", garbageMacroContent);
+ assertFalse(verifyImportUsingSearchPath("/tmp/mytest3a.pig", "mytest3a.pig", "/tmp", false));
}
@Test
@@ -774,12 +751,7 @@ public class TestMacroExpansion {
" $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
"};\n";
- File f = new File("mytest.pig");
- f.deleteOnExit();
-
- FileWriter fw = new FileWriter(f);
- fw.append(macro);
- fw.close();
+ createFile("mytest.pig", macro);
String script =
"import 'mytest.pig';\n" +
@@ -804,28 +776,16 @@ public class TestMacroExpansion {
String macro1 = "define group_and_count (A, reducers) returns B {\n" +
" $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
"};\n";
-
String macro2 = "define distinct_with_reducer(A, reducers) returns B {\n" +
" $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
"};\n";
- File f1 = new File("mytest1.pig");
- f1.deleteOnExit();
-
- FileWriter fw1 = new FileWriter(f1);
- fw1.append(macro1);
- fw1.close();
-
- File f2 = new File("mytest2.pig");
- f2.deleteOnExit();
-
- FileWriter fw2 = new FileWriter(f2);
- fw2.append(macro2);
- fw2.close();
+ createFile("mytest4.pig", macro1);
+ createFile("mytest5.pig", macro2);
String script =
- "import 'mytest1.pig';\n" +
- "import 'mytest2.pig';\n" +
+ "import 'mytest4.pig';\n" +
+ "import 'mytest5.pig';\n" +
"alpha = load 'users' as (user, age, zip);\n" +
"gamma = group_and_count (alpha, 23);\n" +
"beta = distinct_with_reducer(alpha, 32);\n" +
@@ -2004,12 +1964,7 @@ public class TestMacroExpansion {
// PIG-1988
@Test
public void test36() throws Exception {
- File f = new File("mymacro.pig");
- f.deleteOnExit();
-
- FileWriter fw = new FileWriter(f);
- fw.append(" ");
- fw.close();
+ createFile("mymacro.pig", " ");
String query = "import 'mymacro.pig';" +
"define macro1() returns void {}; " +
@@ -2029,17 +1984,8 @@ public class TestMacroExpansion {
// PIG-1987
@Test
public void test37() throws Exception {
- String macro = "define group_and_count (A,group_key, reducers) returns B {\n" +
- " $B = distinct $A partition by org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n" +
- "};\n";
-
- File f = new File("mytest.pig");
- f.deleteOnExit();
-
- FileWriter fw = new FileWriter(f);
- fw.append(macro);
- fw.close();
-
+ createFile("mytest.pig", groupAndCountMacro);
+
String script =
"set default_parallel 10\n" +
"import 'mytest.pig';\n" +
@@ -2124,12 +2070,7 @@ public class TestMacroExpansion {
}
private void verify(String s, String expected) throws Exception {
- File f1 = new File("myscript.pig");
- f1.deleteOnExit();
-
- FileWriter fw1 = new FileWriter(f1);
- fw1.append(s);
- fw1.close();
+ createFile("myscript.pig", s);
String[] args = { "-Dpig.import.search.path=/tmp", "-x", "local", "-c", "myscript.pig" };
PigStats stats = PigRunner.run(args, null);
@@ -2231,5 +2172,54 @@ public class TestMacroExpansion {
private void validateFailure(String piglatin, String expectedErr) throws Throwable {
validateFailure(piglatin, expectedErr, "Reason:");
}
+
+ private void createFile(String filePath, String content) throws Exception {
+ createFile(filePath, content, true);
+ }
+
+ private void createFile(String filePath, String content, boolean deleteOnExit) throws Exception {
+ File f = new File(filePath);
+ if (f.getParent() != null && !(new File(f.getParent())).exists()) {
+ (new File(f.getParent())).mkdirs();
+ }
+
+ if (deleteOnExit) {
+ f.deleteOnExit();
+ }
+
+ FileWriter fw = new FileWriter(f);
+ fw.append(content);
+ fw.close();
+ }
+
+ private boolean verifyImportUsingSearchPath(String macroFilePath, String importFilePath,
+ String importSearchPath) throws Exception {
+ return verifyImportUsingSearchPath(macroFilePath, importFilePath,
+ importSearchPath, true);
+ }
+
+ private boolean verifyImportUsingSearchPath(String macroFilePath, String importFilePath,
+ String importSearchPath, boolean createMacroFilePath) throws Exception {
+
+ if (createMacroFilePath) {
+ createFile(macroFilePath, groupAndCountMacro);
+ }
+
+ String script =
+ "import '" + importFilePath + "';\n" +
+ "alpha = load 'users' as (user, age, zip);\n" +
+ "gamma = group_and_count (alpha, user, 23);\n" +
+ "store gamma into 'byuser';\n";
+
+ createFile("myscript.pig", script);
+
+ String[] args = {
+ (importSearchPath != null ? "-Dpig.import.search.path=" + importSearchPath : ""),
+ "-x", "local", "-c", "myscript.pig"
+ };
+ PigStats stats = PigRunner.run(args, null);
+
+ return stats.isSuccessful();
+ }
}