You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2015/03/20 20:22:47 UTC
svn commit: r1668124 - in /pig/trunk: ./
src/org/apache/pig/backend/hadoop/executionengine/tez/
test/org/apache/pig/test/ test/org/apache/pig/tez/
Author: rohini
Date: Fri Mar 20 19:22:46 2015
New Revision: 1668124
URL: http://svn.apache.org/r1668124
Log:
PIG-4465: Pig streaming ship fails for relative paths on Tez (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
pig/trunk/test/org/apache/pig/test/TestAccumulator.java
pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
pig/trunk/test/org/apache/pig/test/TestSkewedJoin.java
pig/trunk/test/org/apache/pig/test/TestStreaming.java
pig/trunk/test/org/apache/pig/test/Util.java
pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Mar 20 19:22:46 2015
@@ -54,6 +54,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4465: Pig streaming ship fails for relative paths on Tez (rohini)
+
PIG-4461: Use benchmarks for Windows Pig e2e tests (nmaheshwari via daijy)
PIG-4463: AvroMapWrapper still leaks Avro data types and AvroStorageDataConversionUtilities do not handle
Modified: pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java (original)
+++ pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobCompiler.java Fri Mar 20 19:22:46 2015
@@ -92,7 +92,7 @@ public class TezJobCompiler {
String shipFiles = pigContext.getProperties().getProperty("pig.streaming.ship.files");
if (shipFiles != null) {
for (String file : shipFiles.split(",")) {
- TezResourceManager.getInstance().addTezResource(new File(file).toURI());
+ TezResourceManager.getInstance().addTezResource(new File(file.trim()).toURI());
}
}
String cacheFiles = pigContext.getProperties().getProperty("pig.streaming.cache.files");
Modified: pig/trunk/test/org/apache/pig/test/TestAccumulator.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestAccumulator.java?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestAccumulator.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestAccumulator.java Fri Mar 20 19:22:46 2015
@@ -49,7 +49,7 @@ public class TestAccumulator {
private static final String INPUT_FILE2 = "AccumulatorInput2.txt";
private static final String INPUT_FILE3 = "AccumulatorInput3.txt";
private static final String INPUT_FILE4 = "AccumulatorInput4.txt";
- private static final String INPUT_DIR = "build/test/data";
+ private static final String INPUT_DIR = Util.getTestDirectory(TestAccumulator.class);
private static PigServer pigServer;
private static Properties properties;
@@ -88,7 +88,7 @@ public class TestAccumulator {
}
private static void createFiles() throws IOException {
- new File(INPUT_DIR).mkdir();
+ new File(INPUT_DIR).mkdirs();
PrintWriter w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE1));
Modified: pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRegisteredJarVisibility.java Fri Mar 20 19:22:46 2015
@@ -24,12 +24,9 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
-import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
-import java.util.jar.JarFile;
import java.util.jar.JarOutputStream;
import java.util.zip.ZipEntry;
@@ -38,17 +35,9 @@ import javax.tools.JavaFileObject;
import javax.tools.StandardJavaFileManager;
import javax.tools.ToolProvider;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.log4j.FileAppender;
-import org.apache.log4j.Level;
-import org.apache.log4j.Logger;
-import org.apache.log4j.SimpleLayout;
-import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
-import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobControlCompiler;
-import org.apache.pig.data.Tuple;
import org.apache.pig.impl.util.JarManager;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -73,16 +62,15 @@ public class TestRegisteredJarVisibility
private static MiniGenericCluster cluster;
private static File jarFile;
+ private static File testDataDir;
@BeforeClass()
public static void setUp() throws IOException {
String testResourcesDir = "test/resources/" + PACKAGE_NAME.replace(".", "/");
- String testBuildDataDir = "build/test/data";
// Create the test data directory if needed
- File testDataDir = new File(testBuildDataDir,
- TestRegisteredJarVisibility.class.getCanonicalName());
+ testDataDir = new File(Util.getTestDirectory(TestRegisteredJarVisibility.class));
testDataDir.mkdirs();
jarFile = new File(testDataDir, JAR_FILE_NAME);
@@ -111,6 +99,7 @@ public class TestRegisteredJarVisibility
@AfterClass()
public static void tearDown() {
cluster.shutDown();
+ Util.deleteDirectory(testDataDir);
}
@Before
Modified: pig/trunk/test/org/apache/pig/test/TestSkewedJoin.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestSkewedJoin.java?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestSkewedJoin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestSkewedJoin.java Fri Mar 20 19:22:46 2015
@@ -51,7 +51,6 @@ import org.apache.pig.data.Tuple;
import org.apache.pig.data.TupleFactory;
import org.apache.pig.impl.builtin.PartitionSkewedKeys;
import org.apache.pig.impl.logicalLayer.FrontendException;
-import org.apache.pig.newplan.logical.rules.ColumnPruneVisitor;
import org.apache.pig.test.utils.TestHelper;
import org.junit.AfterClass;
import org.junit.Before;
@@ -66,8 +65,9 @@ public class TestSkewedJoin {
private static final String INPUT_FILE5 = "SkewedJoinInput5.txt";
private static final String INPUT_FILE6 = "SkewedJoinInput6.txt";
private static final String INPUT_FILE7 = "SkewedJoinInput7.txt";
- private static final String INPUT_DIR = "build/test/data";
- private static final String OUTPUT_DIR = "build/test/output";
+ private static final String TEST_DIR = Util.getTestDirectory(TestSkewedJoin.class);
+ private static final String INPUT_DIR = TEST_DIR + Path.SEPARATOR + "input";
+ private static final String OUTPUT_DIR = TEST_DIR + Path.SEPARATOR + "output";
private static FileSystem fs;
private static PigServer pigServer;
@@ -97,8 +97,8 @@ public class TestSkewedJoin {
}
private static void createFiles() throws IOException {
- new File(INPUT_DIR).mkdir();
- new File(OUTPUT_DIR).mkdir();
+ new File(INPUT_DIR).mkdirs();
+ new File(OUTPUT_DIR).mkdirs();
PrintWriter w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE1));
@@ -183,8 +183,7 @@ public class TestSkewedJoin {
}
private static void deleteFiles() throws IOException {
- Util.deleteDirectory(new File(INPUT_DIR));
- Util.deleteDirectory(new File(OUTPUT_DIR));
+ Util.deleteDirectory(new File(TEST_DIR));
}
@Test
@@ -556,7 +555,7 @@ public class TestSkewedJoin {
SimpleLayout layout = new SimpleLayout();
FileAppender appender = new FileAppender(layout, logFile.toString(), false, false, 0);
logger.addAppender(appender);
-
+
try {
pigServer.registerScript(new ByteArrayInputStream(script.getBytes("UTF-8")));
pigServer.openIterator("joined");
Modified: pig/trunk/test/org/apache/pig/test/TestStreaming.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestStreaming.java?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestStreaming.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestStreaming.java Fri Mar 20 19:22:46 2015
@@ -40,14 +40,23 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestStreaming {
private static final MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+ private static File testDataDir;
private PigServer pigServer;
+ @BeforeClass
+ public static void oneTimeSetUp() throws Exception {
+ // Create the test data directory if needed
+ testDataDir = new File(Util.getTestDirectory(TestStreaming.class));
+ testDataDir.mkdirs();
+ }
+
@Before
public void setup() throws ExecException {
pigServer = new PigServer(cluster.getExecType(), cluster.getProperties());
@@ -61,6 +70,7 @@ public class TestStreaming {
@AfterClass
public static void oneTimeTearDown() throws Exception {
cluster.shutDown();
+ Util.deleteDirectory(testDataDir);
}
private TupleFactory tf = TupleFactory.getInstance();
@@ -293,7 +303,9 @@ public class TestStreaming {
"}",
};
File command1 = Util.createInputFile("script", "pl", script);
- File command2 = Util.createInputFile("script", "pl", script);
+ // Test relative path
+ File command2 = new File(testDataDir, "testInputShipSpecs.pl");
+ Util.writeToFile(command2, script);
// Expected results
String[] expectedFirstFields =
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Fri Mar 20 19:22:46 2015
@@ -118,6 +118,8 @@ public class Util {
public static final boolean WINDOWS /* borrowed from Path.WINDOWS, Shell.WINDOWS */
= System.getProperty("os.name").startsWith("Windows");
+ public static final String TEST_DIR = System.getProperty("test.build.dir", "build/test");
+
// Helper Functions
// =================
@@ -910,6 +912,7 @@ public class Util {
thread = new Thread (this);
thread.start ();
}
+ @Override
public void run () {
try {
InputStreamReader isr = new InputStreamReader (is);
@@ -1371,22 +1374,28 @@ public class Util {
}
public static Path getFirstPartFile(Path path) throws Exception {
- FileStatus[] parts = FileSystem.get(path.toUri(), new Configuration()).listStatus(path,
+ FileStatus[] parts = FileSystem.get(path.toUri(), new Configuration()).listStatus(path,
new PathFilter() {
@Override
public boolean accept(Path path) {
return path.getName().startsWith("part-");
}
});
- return parts[0].getPath();
+ return parts[0].getPath();
}
public static File getFirstPartFile(File dir) throws Exception {
File[] parts = dir.listFiles(new FilenameFilter() {
+ @Override
public boolean accept(File dir, String name) {
return name.startsWith("part-");
};
});
- return parts[0];
+ return parts[0];
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static String getTestDirectory(Class testClass) {
+ return TEST_DIR + Path.SEPARATOR + "testdata" + Path.SEPARATOR +testClass.getSimpleName();
}
}
Modified: pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java?rev=1668124&r1=1668123&r2=1668124&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java (original)
+++ pig/trunk/test/org/apache/pig/tez/TestTezAutoParallelism.java Fri Mar 20 19:22:46 2015
@@ -52,7 +52,7 @@ import org.junit.Test;
public class TestTezAutoParallelism {
private static final String INPUT_FILE1 = TestTezAutoParallelism.class.getName() + "_1";
private static final String INPUT_FILE2 = TestTezAutoParallelism.class.getName() + "_2";
- private static final String INPUT_DIR = "build/test/data";
+ private static final String INPUT_DIR = Util.getTestDirectory(TestTezAutoParallelism.class);
private static PigServer pigServer;
private static Properties properties;
@@ -85,7 +85,7 @@ public class TestTezAutoParallelism {
}
private static void createFiles() throws IOException {
- new File(INPUT_DIR).mkdir();
+ new File(INPUT_DIR).mkdirs();
PrintWriter w = new PrintWriter(new FileWriter(INPUT_DIR + "/" + INPUT_FILE1));