You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ya...@apache.org on 2010/02/08 23:03:19 UTC
svn commit: r907818 - in /hadoop/pig/branches/branch-0.6/contrib/zebra: ./
src/java/org/apache/hadoop/zebra/io/ src/test/org/apache/hadoop/zebra/io/
src/test/org/apache/hadoop/zebra/mapred/
src/test/org/apache/hadoop/zebra/pig/ src/test/org/apache/hado...
Author: yanz
Date: Mon Feb 8 22:03:19 2010
New Revision: 907818
URL: http://svn.apache.org/viewvc?rev=907818&view=rev
Log:
PIG-1227: Throw exception if column group meta file is missing for an unsorted table (yanz)
Modified:
hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt
hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt Mon Feb 8 22:03:19 2010
@@ -41,6 +41,8 @@
BUG FIXES
+ PIG-1227: Throw exception if column group meta file is missing for an unsorted table (yanz)
+
PIG-1201: unnecessary name node calls by each mapper; too big input split serialization size by Pig's Slice implementation (yanz)
PIG-1167: Hadoop file glob support (yanz)
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java Mon Feb 8 22:03:19 2010
@@ -293,12 +293,8 @@
Path metaFilePath = makeMetaFilePath(path);
/* If index file is not existing */
if (!fs.exists(metaFilePath)) {
- // special case for unsorted CG that did not create index properly.
- if (cgschema.isSorted()) {
- throw new FileNotFoundException(
- "Missing Meta File for sorted Column Group");
- }
- cgindex = buildIndex(fs, path, dirty, conf);
+ throw new FileNotFoundException(
+ "Missing Meta File of " + metaFilePath);
}
else if (cgschema.isSorted()) {
MetaFile.Reader metaFile = MetaFile.createReader(metaFilePath, conf);
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java Mon Feb 8 22:03:19 2010
@@ -353,7 +353,7 @@
@Test
public void testNormalCases() throws IOException, ParseException {
Path path = new Path(rootPath, "TestBasicTableNormal");
- doReadWrite(path, 2, 250, "a, b, c", "", null, "a, d, c, f", false, false);
+ doReadWrite(path, 2, 250, "a, b, c", "", null, "a, d, c, f", true, false);
doReadWrite(path, 2, 250, "a, b, c", "", null, "a, d, c, f", true, false);
doReadWrite(path, 2, 250, "a, b, c", "", "a", "a, d, c, f", true, true);
}
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java Mon Feb 8 22:03:19 2010
@@ -411,7 +411,7 @@
@Test
public void testEmptyCG() throws IOException, ParseException {
Path path = new Path(rootPath, "TestColumnGroupEmptyCG");
- doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", false, false, null);
+ doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", true, false, null);
doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", true, false, null);
doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", true, true, null);
}
@@ -419,14 +419,14 @@
@Test
public void testEmptyTFiles() throws IOException, ParseException {
Path path = new Path(rootPath, "TestColumnGroupEmptyTFile");
- doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", false, false, null);
+ doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", true, false, null);
doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", true, false, null);
doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", true, true, null);
}
public void testNormalCases() throws IOException, ParseException {
Path path = new Path(rootPath, "TestColumnGroupNormal");
- doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", false, false, null);
+ doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", true, false, null);
doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", true, false, null);
doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", true, true, null);
}
@@ -435,7 +435,7 @@
public void testSomeEmptyTFiles() throws IOException, ParseException {
Path path = new Path(rootPath, "TestColumnGroupSomeEmptyTFile");
for (int[] emptyTFiles : new int[][] { { 1, 2 }}) {
- doReadWrite(path, 2, 250, "a, b, c", "a, d, c, f", false, false,
+ doReadWrite(path, 2, 250, "a, b, c", "a, d, c, f", true, false,
emptyTFiles);
doReadWrite(path, 2, 250, "a, b, c", "a, d, c, f", true, false,
emptyTFiles);
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java Mon Feb 8 22:03:19 2010
@@ -67,7 +67,7 @@
@AfterClass
public static void tearDownOnce() throws IOException {
- finish();
+ close();
}
@SuppressWarnings("unchecked")
@@ -175,9 +175,7 @@
ColumnGroup.Writer writer2 = writeOnePart(null, 2);
ColumnGroup.Writer writer3 = writeOnePart(null, 3);
- writer1.finish();
- writer2.finish();
- writer3.finish();
+ writer3.close();
// read in parts
readOnePart(1);
@@ -327,7 +325,7 @@
private static void finish() throws IOException {
if (writer != null) {
- writer.finish();
+ writer.close();
}
}
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java Mon Feb 8 22:03:19 2010
@@ -305,6 +305,7 @@
// set map-only job.
jobConf.setNumReduceTasks(0);
JobClient.runJob(jobConf);
+ BasicTableOutputFormat.close(jobConf);
}
/**
@@ -599,6 +600,7 @@
jobConf.setNumReduceTasks(options.numReducer);
JobClient.runJob(jobConf);
+ BasicTableOutputFormat.close(jobConf);
}
void reduce(Summary sum, Summary delta) {
@@ -950,6 +952,7 @@
jobConf.setNumReduceTasks(1);
JobClient.runJob(jobConf);
+ BasicTableOutputFormat.close(jobConf);
}
void printFreqWords() throws IOException, ParseException {
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java Mon Feb 8 22:03:19 2010
@@ -108,6 +108,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
/*
* create 2nd basic table;
@@ -141,6 +142,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
}
@AfterClass
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java Mon Feb 8 22:03:19 2010
@@ -126,7 +126,8 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
-
+ writer.close();
+
/*
* create 2nd basic table;
*/
@@ -159,7 +160,8 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
-
+ writer.close();
+
/*
* create 3rd basic table;
*/
@@ -192,6 +194,8 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
+
/*
* create 4th basic table;
*/
@@ -224,6 +228,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
/*
* create 5th basic table;
*/
@@ -256,7 +261,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
-
+ writer.close();
}
@AfterClass
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java Mon Feb 8 22:03:19 2010
@@ -118,6 +118,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
}
@AfterClass
@@ -137,4 +138,4 @@
System.out.println(cur);
}
}
-}
\ No newline at end of file
+}
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java Mon Feb 8 22:03:19 2010
@@ -159,6 +159,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
}
@AfterClass
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java Mon Feb 8 22:03:19 2010
@@ -114,6 +114,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
}
@AfterClass
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java Mon Feb 8 22:03:19 2010
@@ -103,6 +103,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
}
@AfterClass
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java Mon Feb 8 22:03:19 2010
@@ -149,6 +149,7 @@
for (int i = 0; i < numsInserters; i++) {
inserters[i].close();
}
+ writer.close();
}
@AfterClass
Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java Mon Feb 8 22:03:19 2010
@@ -359,9 +359,10 @@
} catch (Exception e) {
e.printStackTrace();
}
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
+
BasicTable.dumpInfo(path1.toString(), ps, conf);
System.out.println("start dumpinfo ===========\n" + bos.toString());
Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
@@ -382,7 +383,7 @@
} catch (Exception e) {
e.printStackTrace();
}
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -404,7 +405,7 @@
} catch (Exception e) {
e.printStackTrace();
}
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -426,7 +427,7 @@
} catch (Exception e) {
e.printStackTrace();
}
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -448,7 +449,7 @@
} catch (Exception e) {
e.printStackTrace();
}
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -470,7 +471,7 @@
} catch (Exception e) {
e.printStackTrace();
}
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -491,7 +492,7 @@
} catch (Exception e) {
e.printStackTrace();
}
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -605,7 +606,7 @@
fs = path.getFileSystem(conf);
BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
conf);
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
System.out.println("start dumpinfo 17 ===========");
@@ -699,7 +700,7 @@
fs = path.getFileSystem(conf);
BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
conf);
- writer.finish();
+ writer.close();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
PrintStream ps = new PrintStream(bos);
System.out.println("start dumpinfo 22===========" + bos.toString());