You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/07/09 20:06:51 UTC
svn commit: r962621 - in /hadoop/pig/trunk: CHANGES.txt
src/org/apache/pig/builtin/BinStorage.java
test/org/apache/pig/test/TestEvalPipeline2.java
Author: daijy
Date: Fri Jul 9 18:06:51 2010
New Revision: 962621
URL: http://svn.apache.org/viewvc?rev=962621&view=rev
Log:
PIG-1484: BinStorage should support comma seperated path
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java
hadoop/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=962621&r1=962620&r2=962621&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Jul 9 18:06:51 2010
@@ -95,6 +95,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1484: BinStorage should support comma seperated path (daijy)
+
PIG-1469: DefaultDataBag assumes ArrayList as default List type (azaroth via dvryaboy)
PIG-1467: order by fail when set "fs.file.impl.disable.cache" to true (daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java?rev=962621&r1=962620&r2=962621&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java Fri Jul 9 18:06:51 2010
@@ -396,12 +396,17 @@ implements LoadCaster, StoreFuncInterfac
// we can treat either local or hadoop mode as hadoop mode - hence
// we can use HDataStorage and FileLocalizer.openDFSFile below
HDataStorage storage = new HDataStorage(props);
- if (!FileLocalizer.fileExists(location, storage)) {
- // At compile time in batch mode, the file may not exist
- // (such as intermediate file). Just return null - the
- // same way as we would if we did not get a valid record
- return null;
+
+ // At compile time in batch mode, the file may not exist
+ // (such as intermediate file). Just return null - the
+ // same way as we would if we did not get a valid record
+ String[] locations = getPathStrings(location);
+ for (String loc : locations) {
+ if (!FileLocalizer.fileExists(loc, storage)) {
+ return null;
+ }
}
+
ReadToEndLoader loader = new ReadToEndLoader(this, conf, location, 0);
// get the first record from the input file
// and figure out the schema from the data in
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java
URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java?rev=962621&r1=962620&r2=962621&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestEvalPipeline2.java Fri Jul 9 18:06:51 2010
@@ -672,4 +672,44 @@ public class TestEvalPipeline2 extends T
}
}
+ // See PIG-972
+ @Test
+ public void testBinStorageCommaSeperatedPath() throws Exception{
+ String[] input = {
+ "1\t3",
+ "2\t4",
+ "3\t5"
+ };
+
+ Util.createInputFile(cluster, "table_simple1", input);
+ pigServer.setBatchOn();
+ pigServer.registerQuery("A = LOAD 'table_simple1' as (a0, a1);");
+ pigServer.registerQuery("store A into 'table_simple1.bin' using BinStorage();");
+ pigServer.registerQuery("store A into 'table_simple2.bin' using BinStorage();");
+
+ pigServer.executeBatch();
+
+ pigServer.registerQuery("A = LOAD 'table_simple1.bin,table_simple2.bin' using BinStorage();");
+ Iterator<Tuple> iter = pigServer.openIterator("A");
+
+ Tuple t = iter.next();
+ assertTrue(t.toString().equals("(1,3)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(2,4)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(3,5)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(1,3)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(2,4)"));
+
+ t = iter.next();
+ assertTrue(t.toString().equals("(3,5)"));
+
+ assertFalse(iter.hasNext());
+ }
}