You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ch...@apache.org on 2014/05/30 03:10:48 UTC
svn commit: r1598448 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/impl/io/FileLocalizer.java
test/org/apache/pig/test/TestPigServer.java
Author: cheolsoo
Date: Fri May 30 01:10:48 2014
New Revision: 1598448
URL: http://svn.apache.org/r1598448
Log:
PIG-3874: FileLocalizer temp path can sometimes be non-unique (chitnis via cheolsoo)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
pig/trunk/test/org/apache/pig/test/TestPigServer.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1598448&r1=1598447&r2=1598448&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 30 01:10:48 2014
@@ -159,6 +159,8 @@ PIG-3882: Multiquery off mode execution
BUG FIXES
+PIG-3874: FileLocalizer temp path can sometimes be non-unique (chitnis via cheolsoo)
+
PIG-3967: Grunt fail if we running more statement after first store (daijy)
PIG-3915: MapReduce queries in Pigmix outputs different results than Pig's (keren3000 via daijy)
Modified: pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java?rev=1598448&r1=1598447&r2=1598448&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java (original)
+++ pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java Fri May 30 01:10:48 2014
@@ -429,7 +429,7 @@ public class FileLocalizer {
return true;
}
- static Random r = new Random();
+ static Random r = new Random();
/**
* Thread local relativeRoot ContainerDescriptor. Do not access this object
@@ -464,14 +464,25 @@ public class FileLocalizer {
if (relativeRoot.get() == null) {
String tdir= Utils.substituteVars(pigContext.getProperties().getProperty(PigConfiguration.PIG_TEMP_DIR, "/tmp"));
- ContainerDescriptor relative = pigContext.getDfs().asContainer(tdir + "/temp" + r.nextInt());
- relativeRoot.set(relative);
+ ContainerDescriptor relative;
try {
- if (!relative.exists()) {
+ do {
+ relative = pigContext.getDfs().asContainer(tdir + "/temp" + r.nextInt());
+ } while (relative.exists());
+ relativeRoot.set(relative);
+ createRelativeRoot(relative);
+ }
+ catch (IOException e) {
+ // try one last time in case this was due IO Exception caused by dir
+ // operations on directory created by another JVM at the same instant
+ relative = pigContext.getDfs().asContainer(tdir + "/temp" + r.nextInt());
+ relativeRoot.set(relative);
+ try {
createRelativeRoot(relative);
}
- } catch (IOException e) {
- throw new DataStorageException(e);
+ catch (IOException e1) {
+ throw new DataStorageException(e1);
+ }
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=1598448&r1=1598447&r2=1598448&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigServer.java Fri May 30 01:10:48 2014
@@ -46,6 +46,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.Random;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -55,7 +56,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.ExecType;
-import org.apache.pig.PigConfiguration;
import org.apache.pig.PigServer;
import org.apache.pig.ResourceSchema;
import org.apache.pig.backend.executionengine.ExecException;
@@ -121,7 +121,7 @@ public class TestPigServer {
fail("Included is false, but url ["+url+"] contains name ["+name+"]");
}
assertEquals("Too many urls contain name: " + name, 1, ++count);
- }
+ }
}
if (included) {
assertEquals("Number of urls that contain name [" + name + "] != 1", 1, count);
@@ -775,24 +775,48 @@ public class TestPigServer {
@Test
public void testPigTempDir() throws Throwable {
- File propertyFile = new File(tempDir, "pig.properties");
- propertyFile.deleteOnExit();
- PrintWriter out = new PrintWriter(new FileWriter(propertyFile));
- out.println(PigConfiguration.PIG_TEMP_DIR + "=/tmp/test");
- out.close();
Properties properties = PropertiesUtil.loadDefaultProperties();
+ File pigTempDir = new File(tempDir, FILE_SEPARATOR + "tmp" + FILE_SEPARATOR + "test");
+ properties.put("pig.temp.dir", pigTempDir.getPath());
PigContext pigContext=new PigContext(ExecType.LOCAL, properties);
pigContext.connect();
FileLocalizer.setInitialized(false);
+
String tempPath= FileLocalizer.getTemporaryPath(pigContext).toString();
- assertTrue(tempPath.startsWith("file:/tmp/test/"));
Path path = new Path(tempPath);
+ assertTrue(tempPath.startsWith(pigTempDir.toURI().toString()));
+
FileSystem fs = FileSystem.get(path.toUri(),
ConfigurationUtil.toConfiguration(pigContext.getProperties()));
FileStatus status = fs.getFileStatus(path.getParent());
// Temporary root dir should have 700 as permission
assertEquals("rwx------", status.getPermission().toString());
- propertyFile.delete();
+ pigTempDir.delete();
+ FileLocalizer.setInitialized(false);
+ }
+
+ @Test
+ public void testUniquePigTempDir() throws Throwable {
+ Properties properties = PropertiesUtil.loadDefaultProperties();
+ File pigTempDir = new File(tempDir, FILE_SEPARATOR + "tmp" + FILE_SEPARATOR + "test");
+ properties.put("pig.temp.dir", pigTempDir.getPath());
+ PigContext pigContext = new PigContext(ExecType.LOCAL, properties);
+ pigContext.connect();
+ FileLocalizer.setInitialized(false);
+
+ Random r = new Random(5);
+ FileLocalizer.setR(r);
+ String tempPath1 = FileLocalizer.getTemporaryPath(pigContext).toString();
+
+ FileLocalizer.setInitialized(false);
+ r = new Random(5);
+ FileLocalizer.setR(r);
+ String tempPath2 = FileLocalizer.getTemporaryPath(pigContext).toString();
+
+ assertFalse(tempPath1.toString().equals(tempPath2.toString()));
+
+ // cleanup
+ pigTempDir.delete();
FileLocalizer.setInitialized(false);
}