You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by ro...@apache.org on 2017/06/29 22:36:34 UTC
svn commit: r1800339 - in /pig/trunk: ./
contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/
contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/
src/org/apache/pig/ src/org/apache/pig/builtin/ test/o...
Author: rohini
Date: Thu Jun 29 22:36:33 2017
New Revision: 1800339
URL: http://svn.apache.org/viewvc?rev=1800339&view=rev
Log:
PIG-5263: Using wildcard doesn't work with OrcStorage (satishsaley via rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java
pig/trunk/src/org/apache/pig/LoadFunc.java
pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
pig/trunk/test/org/apache/pig/test/TestLoadFunc.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jun 29 22:36:33 2017
@@ -32,6 +32,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5263: Using wildcard doesn't work with OrcStorage (satishsaley via rohini)
+
PIG-4548: Records Lost With Specific Combination of Commands and Streaming Function (knoguchi)
PIG-5262: Fix jdiff related issues: fail build upon error, correct xml character escaping (szita)
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorage.java Thu Jun 29 22:36:33 2017
@@ -61,6 +61,7 @@ import org.apache.pig.builtin.FuncUtils;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
@@ -190,7 +191,7 @@ public class AvroStorage extends FileInp
if (inputAvroSchema == null || UDFContext.getUDFContext().isFrontend()) {
Configuration conf = job.getConfiguration();
- Set<Path> paths = AvroStorageUtils.getPaths(location, conf, true);
+ Set<Path> paths = getGlobPaths(location, conf, true);
if (!paths.isEmpty()) {
// Set top level directories in input format. Adding all files will
// bloat configuration size
@@ -273,7 +274,7 @@ public class AvroStorage extends FileInp
*/
@SuppressWarnings("deprecation")
protected Schema getAvroSchema(Path path, FileSystem fs) throws IOException {
- if (!fs.exists(path) || !AvroStorageUtils.PATH_FILTER.accept(path))
+ if (!fs.exists(path) || !Utils.VISIBLE_FILES.accept(path))
return null;
/* if path is first level directory or is a file */
@@ -281,7 +282,7 @@ public class AvroStorage extends FileInp
return getSchema(path, fs);
}
- FileStatus[] ss = fs.listStatus(path, AvroStorageUtils.PATH_FILTER);
+ FileStatus[] ss = fs.listStatus(path, Utils.VISIBLE_FILES);
Schema schema = null;
if (ss.length > 0) {
if (AvroStorageUtils.noDir(ss))
@@ -433,7 +434,7 @@ public class AvroStorage extends FileInp
// getSchema is called during script parsing we don't want to fail
// here if path not found
- Set<Path> paths = AvroStorageUtils.getPaths(location, conf, false);
+ Set<Path> paths = getGlobPaths(location, conf, false);
if (!paths.isEmpty()) {
setInputAvroSchema(paths, conf);
}
Modified: pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java (original)
+++ pig/trunk/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/avro/AvroStorageUtils.java Thu Jun 29 22:36:33 2017
@@ -41,6 +41,7 @@ import org.apache.pig.LoadFunc;
import org.apache.pig.ResourceSchema;
import org.apache.pig.ResourceSchema.ResourceFieldSchema;
import org.apache.pig.data.DataType;
+import org.apache.pig.impl.util.Utils;
import org.codehaus.jackson.JsonNode;
/**
* This is utility class for this package
@@ -59,15 +60,6 @@ public class AvroStorageUtils {
private static final String NONAME = "NONAME";
private static final String PIG_TUPLE_WRAPPER = "PIG_WRAPPER";
- /** ignore hdfs files with prefix "_" and "." */
- public static PathFilter PATH_FILTER = new PathFilter() {
- @Override
- public boolean accept(Path path) {
- return !path.getName().startsWith("_")
- && !path.getName().startsWith(".");
- }
- };
-
static String getDummyFieldName(int index) {
return NONAME + "_" + index;
}
@@ -93,33 +85,6 @@ public class AvroStorageUtils {
}
/**
- * Gets the list of paths from the pathString specified which may contain
- * comma-separated paths and glob style path
- *
- * @throws IOException
- */
- public static Set<Path> getPaths(String pathString, Configuration conf, boolean failIfNotFound)
- throws IOException {
- Set<Path> paths = new HashSet<Path>();
- String[] pathStrs = LoadFunc.getPathStrings(pathString);
- for (String pathStr : pathStrs) {
- FileSystem fs = FileSystem.get(new Path(pathStr).toUri(), conf);
- FileStatus[] matchedFiles = fs.globStatus(new Path(pathStr), PATH_FILTER);
- if (matchedFiles == null || matchedFiles.length == 0) {
- if (failIfNotFound) {
- throw new IOException("Input Pattern " + pathStr + " matches 0 files");
- } else {
- continue;
- }
- }
- for (FileStatus file : matchedFiles) {
- paths.add(file.getPath());
- }
- }
- return paths;
- }
-
- /**
* Returns all non-hidden files recursively inside the base paths given
*
* @throws IOException
@@ -140,7 +105,7 @@ public class AvroStorageUtils {
private static void getAllFilesInternal(FileStatus file, Configuration conf,
Set<Path> paths, FileSystem fs) throws IOException {
- for (FileStatus f : fs.listStatus(file.getPath(), PATH_FILTER)) {
+ for (FileStatus f : fs.listStatus(file.getPath(), Utils.VISIBLE_FILES)) {
if (f.isDir()) {
getAllFilesInternal(f, conf, paths, fs);
} else {
@@ -167,7 +132,7 @@ public class AvroStorageUtils {
if (!status.isDir()) {
return path;
}
- FileStatus[] statuses = fs.listStatus(path, PATH_FILTER);
+ FileStatus[] statuses = fs.listStatus(path, Utils.VISIBLE_FILES);
if (statuses.length == 0) {
return null;
Modified: pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java
URL: http://svn.apache.org/viewvc/pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java (original)
+++ pig/trunk/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/avro/TestAvroStorageUtils.java Thu Jun 29 22:36:33 2017
@@ -89,54 +89,6 @@ public class TestAvroStorageUtils {
assertNull(realSchema);
}
- @Test
- public void testGetPaths() throws IOException {
- final String basedir = "file://" + System.getProperty("user.dir");
- final String tempdir = Long.toString(System.currentTimeMillis());
- final String nonexistentpath = basedir + "/" + tempdir + "/this_path_does_not_exist";
-
- String locationStr = null;
- Set<Path> paths;
- Configuration conf = new Configuration();
-
- // existent path
- locationStr = basedir;
- paths = AvroStorageUtils.getPaths(locationStr, conf, true);
- assertFalse(paths.isEmpty());
-
- // non-existent path
- locationStr = nonexistentpath;
- try {
- paths = AvroStorageUtils.getPaths(locationStr, conf, true);
- fail();
- } catch (IOException e) {
- assertTrue(e.getMessage().contains("matches 0 files"));
- }
-
- // empty glob pattern
- locationStr = basedir + "/{}";
- try {
- paths = AvroStorageUtils.getPaths(locationStr, conf, true);
- fail();
- } catch (IOException e) {
- assertTrue(e.getMessage().contains("matches 0 files"));
- }
-
- paths = AvroStorageUtils.getPaths(locationStr, conf, false);
- assertTrue(paths.isEmpty());
-
- // bad glob pattern
- locationStr = basedir + "/{1,";
- try {
- AvroStorageUtils.getPaths(locationStr, conf, true);
- Assert.fail("Negative test to test illegal file pattern. Should not be succeeding!");
- } catch (IOException e) {
- // The message of the exception for illegal file pattern is rather long,
- // so we simply confirm if it contains 'illegal file pattern'.
- assertTrue(e.getMessage().contains("Illegal file pattern"));
- }
- }
-
// test merging null and non-null
@Test
public void testMergeSchema1() throws IOException {
Modified: pig/trunk/src/org/apache/pig/LoadFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/LoadFunc.java?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/LoadFunc.java (original)
+++ pig/trunk/src/org/apache/pig/LoadFunc.java Thu Jun 29 22:36:33 2017
@@ -21,9 +21,14 @@ import java.io.IOException;
import java.net.URI;
import java.util.AbstractCollection;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.InputFormat;
@@ -38,6 +43,7 @@ import org.apache.pig.builtin.Utf8Storag
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.logicalLayer.FrontendException;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.impl.util.Utils;
import org.apache.pig.tools.pigstats.PigStatusReporter;
@@ -210,7 +216,38 @@ public abstract class LoadFunc {
return pathStrings.toArray(new String[0]);
}
-
+
+ /**
+ * Return all the file paths in commaSeparatedPaths matching patterns if any
+ *
+ * @param commaSeparatedPaths
+ * @param conf
+ * @param failIfNotFound
+ * @return a set of paths
+ * @throws IOException
+ */
+ public static Set<Path> getGlobPaths(String commaSeparatedPaths, Configuration conf, boolean failIfNotFound)
+ throws IOException {
+ Set<Path> paths = new HashSet<Path>();
+ String[] pathStrs = LoadFunc.getPathStrings(commaSeparatedPaths);
+ for (String pathStr : pathStrs) {
+ FileSystem fs = FileSystem.get(new Path(pathStr).toUri(), conf);
+ FileStatus[] matchedFiles = fs.globStatus(new Path(pathStr), Utils.VISIBLE_FILES);
+ if (matchedFiles == null || matchedFiles.length == 0) {
+ if (failIfNotFound) {
+ throw new IOException("Input Pattern " + pathStr + " matches 0 files");
+ }
+ else {
+ continue;
+ }
+ }
+ for (FileStatus file : matchedFiles) {
+ paths.add(file.getPath());
+ }
+ }
+ return paths;
+ }
+
/**
* Construct the absolute path from the file location and the current
* directory. The current directory is either of the form
@@ -324,4 +361,5 @@ public abstract class LoadFunc {
public List<String> getShipFiles() {
return null;
}
+
}
Modified: pig/trunk/src/org/apache/pig/builtin/OrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/OrcStorage.java?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/builtin/OrcStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/OrcStorage.java Thu Jun 29 22:36:33 2017
@@ -25,6 +25,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -315,9 +316,14 @@ public class OrcStorage extends LoadFunc
if (p.getProperty(signature + SearchArgsSuffix) != null) {
job.getConfiguration().set(SARG_PUSHDOWN, p.getProperty(signature + SearchArgsSuffix));
}
-
}
- FileInputFormat.setInputPaths(job, location);
+ Set<Path> paths = getGlobPaths(location, job.getConfiguration(), true);
+ if (!paths.isEmpty()) {
+ FileInputFormat.setInputPaths(job, paths.toArray(new Path[paths.size()]));
+ }
+ else {
+ throw new IOException("Input path \'" + location + "\' is not found");
+ }
}
private String getReqiredColumnIdString(boolean[] requiredColumns) {
Modified: pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java Thu Jun 29 22:36:33 2017
@@ -125,7 +125,7 @@ public class TestOrcStorage {
@Test
public void testSimpleLoad() throws Exception {
- pigServer.registerQuery("A = load '" + basedir + "orc-file-11-format.orc'" + " using OrcStorage();" );
+ pigServer.registerQuery("A = load '" + basedir + "*-file-11-format.orc'" + " using OrcStorage();");
Schema s = pigServer.dumpSchema("A");
assertEquals(s.toString(), "{boolean1: boolean,byte1: int,short1: int,int1: int,long1: long," +
"float1: float,double1: double,bytes1: bytearray,string1: chararray," +
Modified: pig/trunk/test/org/apache/pig/test/TestLoadFunc.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestLoadFunc.java?rev=1800339&r1=1800338&r2=1800339&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestLoadFunc.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestLoadFunc.java Thu Jun 29 22:36:33 2017
@@ -19,10 +19,15 @@
package org.apache.pig.test;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
import java.io.IOException;
+import java.util.Set;
import org.junit.Assert;
-
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.pig.LoadFunc;
import org.apache.pig.impl.logicalLayer.FrontendException;
@@ -150,6 +155,57 @@ public class TestLoadFunc {
Assert.assertEquals("har:///user/pig/harfile",
LoadFunc.getAbsolutePath("har:///user/pig/harfile",
curHdfsDir));
- }
-
+ }
+
+ @Test
+ public void testGlobPaths() throws IOException {
+ final String basedir = "file://" + System.getProperty("user.dir");
+ final String tempdir = Long.toString(System.currentTimeMillis());
+ final String nonexistentpath = basedir + "/" + tempdir + "/this_path_does_not_exist";
+
+ String locationStr = null;
+ Set<Path> paths;
+ Configuration conf = new Configuration();
+
+ // existent path
+ locationStr = basedir;
+ paths = LoadFunc.getGlobPaths(locationStr, conf, true);
+ assertFalse(paths.isEmpty());
+
+ // non-existent path
+ locationStr = nonexistentpath;
+ try {
+ paths = LoadFunc.getGlobPaths(locationStr, conf, true);
+ fail("Paths with pattern are not readable");
+ }
+ catch (IOException e) {
+ assertTrue(e.getMessage().contains("matches 0 files"));
+ }
+
+ // empty glob pattern
+ locationStr = basedir + "/{}";
+ try {
+ paths = LoadFunc.getGlobPaths(locationStr, conf, true);
+ fail();
+ }
+ catch (IOException e) {
+ assertTrue(e.getMessage().contains("matches 0 files"));
+ }
+
+ paths = LoadFunc.getGlobPaths(locationStr, conf, false);
+ assertTrue(paths.isEmpty());
+
+ // bad glob pattern
+ locationStr = basedir + "/{1,";
+ try {
+ LoadFunc.getGlobPaths(locationStr, conf, true);
+ Assert.fail("Negative test to test illegal file pattern. Should not be succeeding!");
+ }
+ catch (IOException e) {
+ // The message of the exception for illegal file pattern is rather
+ // long, so we simply confirm if it contains 'illegal file pattern'.
+ assertTrue(e.getMessage().contains("Illegal file pattern"));
+ }
+ }
+
}