You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by sm...@apache.org on 2012/08/06 07:18:41 UTC
svn commit: r1369723 - in /pig/trunk: CHANGES.txt
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Author: sms
Date: Mon Aug 6 05:18:41 2012
New Revision: 1369723
URL: http://svn.apache.org/viewvc?rev=1369723&view=rev
Log:
PIG-2856: AvroStorage doesn't load files in the directories when a glob pattern matches both files and directories. (cheolsoo via sms)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1369723&r1=1369722&r2=1369723&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Aug 6 05:18:41 2012
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-2856: AvroStorage doesn't load files in the directories when a glob pattern matches both files and directories. (cheolsoo via sms)
+
PIG-2569: Fix org.apache.pig.test.TestInvoker.testSpeed (aklochkov via dvryaboy)
PIG-2858: Improve PlanHelper to allow finding any PhysicalOperator in a plan (dvryaboy)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java?rev=1369723&r1=1369722&r2=1369723&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java Mon Aug 6 05:18:41 2012
@@ -124,7 +124,7 @@ public class AvroStorageUtils {
}
for (FileStatus file : matchedFiles) {
if (file.isDir()) {
- for (FileStatus sub : fs.listStatus(path)) {
+ for (FileStatus sub : fs.listStatus(file.getPath())) {
getAllSubDirs(sub.getPath(), job, paths);
}
} else {
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java?rev=1369723&r1=1369722&r2=1369723&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorage.java Mon Aug 6 05:18:41 2012
@@ -29,6 +29,8 @@ import org.apache.hadoop.mapreduce.Job;
import org.apache.pig.ExecType;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.backend.executionengine.ExecJob;
+import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.JobCreationException;
import org.apache.pig.piggybank.storage.avro.AvroStorage;
import org.apache.pig.piggybank.storage.avro.PigSchema2Avro;
@@ -41,6 +43,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
+import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -112,7 +115,7 @@ public class TestAvroStorage {
public void testGlob1() throws IOException {
// Verify that the a glob pattern matches files properly.
String output = outbasedir + "testGlob1";
- String expected = basedir + "expected_test_dir_1.avro";
+ String expected = basedir + "expected_testDir.avro";
deleteDirectory(new File(output));
String [] queries = {
" in = LOAD '" + testDir1AllFiles + "' USING org.apache.pig.piggybank.storage.avro.AvroStorage ();",
@@ -492,11 +495,14 @@ public class TestAvroStorage {
if (query != null && query.length() > 0)
pigServerLocal.registerQuery(query);
}
- pigServerLocal.executeBatch();
+ List<ExecJob> jobs = pigServerLocal.executeBatch();
+ for (ExecJob job : jobs) {
+ assertEquals(JOB_STATUS.COMPLETED, job.getStatus());
+ }
}
private void verifyResults(String outPath, String expectedOutpath) throws IOException {
- verifyResults(outPath, expectedOutpath, null);
+ verifyResults(outPath, expectedOutpath, null);
}
private void verifyResults(String outPath, String expectedOutpath, String expectedCodec) throws IOException {