You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2016/07/17 08:25:57 UTC
svn commit: r1753026 - in /pig/branches/branch-0.16: CHANGES.txt
src/org/apache/pig/Main.java src/org/apache/pig/tools/pigstats/PigStats.java
test/org/apache/pig/test/TestPigRunner.java
test/org/apache/pig/test/Util.java
Author: daijy
Date: Sun Jul 17 08:25:57 2016
New Revision: 1753026
URL: http://svn.apache.org/viewvc?rev=1753026&view=rev
Log:
PIG-4950: Fix minor issues with running scripts in non-local FileSystems
Modified:
pig/branches/branch-0.16/CHANGES.txt
pig/branches/branch-0.16/src/org/apache/pig/Main.java
pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java
pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java
pig/branches/branch-0.16/test/org/apache/pig/test/Util.java
Modified: pig/branches/branch-0.16/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/CHANGES.txt?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/CHANGES.txt (original)
+++ pig/branches/branch-0.16/CHANGES.txt Sun Jul 17 08:25:57 2016
@@ -32,6 +32,8 @@ BUG FIXES
PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16 (yangyishan0901m via daijy)
+PIG-4950: Fix minor issues with running scripts in non-local FileSystems (petersla via daijy)
+
Release 0.16.0
INCOMPATIBLE CHANGES
Modified: pig/branches/branch-0.16/src/org/apache/pig/Main.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/Main.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/Main.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/Main.java Sun Jul 17 08:25:57 2016
@@ -477,7 +477,7 @@ public class Main {
}
- logFileName = validateLogFile(logFileName, file);
+ logFileName = validateLogFile(logFileName, localFileRet.file);
pigContext.getProperties().setProperty("pig.logfile", (logFileName == null? "": logFileName));
// Set job name based on name of the script
@@ -488,7 +488,7 @@ public class Main {
new File(substFile).deleteOnExit();
}
- scriptState.setScript(new File(file));
+ scriptState.setScript(localFileRet.file);
grunt = new Grunt(pin, pigContext);
gruntCalled = true;
@@ -605,7 +605,7 @@ public class Main {
return ReturnCode.SUCCESS;
}
- logFileName = validateLogFile(logFileName, remainders[0]);
+ logFileName = validateLogFile(logFileName, localFileRet.file);
pigContext.getProperties().setProperty("pig.logfile", (logFileName == null? "": logFileName));
if (!debug) {
@@ -971,11 +971,10 @@ public class Main {
System.out.println("Additionally, any Hadoop property can be specified.");
}
- private static String validateLogFile(String logFileName, String scriptName) {
+ private static String validateLogFile(String logFileName, File scriptFile) {
String strippedDownScriptName = null;
- if(scriptName != null) {
- File scriptFile = new File(scriptName);
+ if (scriptFile != null) {
if(!scriptFile.isDirectory()) {
String scriptFileAbsPath;
try {
Modified: pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java (original)
+++ pig/branches/branch-0.16/src/org/apache/pig/tools/pigstats/PigStats.java Sun Jul 17 08:25:57 2016
@@ -265,6 +265,13 @@ public abstract class PigStats {
return ScriptState.get().getPigVersion();
}
+ /**
+ * Returns the contents of the script that was run.
+ */
+ public String getScript() {
+ return ScriptState.get().getScript();
+ }
+
public String getScriptId() {
return ScriptState.get().getId();
}
Modified: pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/test/TestPigRunner.java Sun Jul 17 08:25:57 2016
@@ -19,6 +19,7 @@ package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -35,6 +36,8 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.Counters;
@@ -59,7 +62,9 @@ import org.junit.AfterClass;
import org.junit.Assume;
import org.junit.Before;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
public class TestPigRunner {
@@ -70,6 +75,9 @@ public class TestPigRunner {
private static final String OUTPUT_FILE = "output";
private static final String PIG_FILE = "test.pig";
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
@BeforeClass
public static void setUpBeforeClass() throws Exception {
cluster = MiniGenericCluster.buildCluster();
@@ -1110,6 +1118,77 @@ public class TestPigRunner {
Util.deleteFile(cluster, "tmp/output");
}
}
+
+ @Test
+ public void testStoredScriptContents() throws Exception {
+ String scriptContents = "sh echo success;\n";
+ FileUtils.writeStringToFile(new File(PIG_FILE), scriptContents);
+ Util.copyFromLocalToCluster(cluster, PIG_FILE, PIG_FILE);
+
+ Path inputInDfs = new Path(cluster.getFileSystem().getHomeDirectory(), PIG_FILE);
+ try {
+ runAndValidateStoredScriptContents(PIG_FILE, scriptContents);
+ runAndValidateStoredScriptContents(inputInDfs.toString(), scriptContents);
+ } finally {
+ FileUtils.deleteQuietly(new File(PIG_FILE));
+ Util.deleteQuietly(cluster, PIG_FILE);
+ }
+ }
+
+ @Test
+ public void testErrorLogUnderCustomDir() throws Exception {
+ try (PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE))) {
+ w.println("A = load '" + INPUT_FILE + "' as (a0:int, a1:int, a2:int);");
+ w.println("B = foreach A generate StringSize(a0);");
+ w.println("store B into '" + OUTPUT_FILE + "';");
+ }
+ Util.copyFromLocalToCluster(cluster, PIG_FILE, PIG_FILE);
+
+ Path inputInDfs = new Path(cluster.getFileSystem().getHomeDirectory(), PIG_FILE);
+ try {
+ runAndValidateCustomErrorLogDir(PIG_FILE);
+ runAndValidateCustomErrorLogDir(inputInDfs.toString());
+ } finally {
+ FileUtils.deleteQuietly(new File(PIG_FILE));
+ Util.deleteQuietly(cluster, PIG_FILE);
+ }
+ }
+
+ private void runAndValidateStoredScriptContents(String scriptPath, String expectedContents) {
+ PigStats stats = runPigLocally(scriptPath);
+ assertTrue(stats.isSuccessful());
+ assertEquals(expectedContents, stats.getScript());
+
+ stats = runPigLocally("-f", scriptPath);
+ assertTrue(stats.isSuccessful());
+ assertEquals(expectedContents, stats.getScript());
+ }
+
+ private void runAndValidateCustomErrorLogDir(String scriptPath) throws IOException {
+ File logsFolder = temporaryFolder.newFolder();
+ String logsPath = logsFolder.getAbsolutePath();
+ assertFileCountUnderDir(logsFolder, 0);
+
+ PigStats stats = runPigLocally("-l", logsPath, scriptPath);
+ assertFalse(stats.isSuccessful());
+ assertFileCountUnderDir(logsFolder, 1);
+
+ stats = runPigLocally("-l", logsPath, "-f", scriptPath);
+ assertFalse(stats.isSuccessful());
+ assertFileCountUnderDir(logsFolder, 2);
+ }
+
+ private void assertFileCountUnderDir(File directory, int expectedFileCount) throws IOException {
+ String[] files = directory.list();
+ assertNotNull(files);
+ assertEquals(expectedFileCount, files.length);
+ }
+
+ private PigStats runPigLocally(String... extraArgs) {
+ String[] args = ArrayUtils.addAll(new String[]{"-x", "local"}, extraArgs);
+ return PigRunner.run(args, new TestNotificationListener("local"));
+ }
+
public static class TestNotificationListener implements PigProgressNotificationListener {
private Map<String, int[]> numMap = new HashMap<String, int[]>();
Modified: pig/branches/branch-0.16/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/branches/branch-0.16/test/org/apache/pig/test/Util.java?rev=1753026&r1=1753025&r2=1753026&view=diff
==============================================================================
--- pig/branches/branch-0.16/test/org/apache/pig/test/Util.java (original)
+++ pig/branches/branch-0.16/test/org/apache/pig/test/Util.java Sun Jul 17 08:25:57 2016
@@ -480,6 +480,19 @@ public class Util {
fs.delete(new Path(fileName), true);
}
+ /**
+ * Deletes a dfs file from the MiniCluster DFS quietly
+ *
+ * @param miniCluster the MiniCluster where the file should be deleted
+ * @param fileName the path of the file to be deleted
+ */
+ public static void deleteQuietly(MiniGenericCluster miniCluster, String fileName) {
+ try {
+ deleteFile(miniCluster, fileName);
+ } catch (IOException ignored) {
+ }
+ }
+
static public void deleteFile(PigContext pigContext, String fileName)
throws IOException {
Configuration conf = ConfigurationUtil.toConfiguration(