You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2016/07/16 05:05:34 UTC
svn commit: r1752888 - in /pig/trunk: CHANGES.txt
src/org/apache/pig/parser/RegisterResolver.java
test/org/apache/pig/test/TestRegisterParser.java
Author: daijy
Date: Sat Jul 16 05:05:34 2016
New Revision: 1752888
URL: http://svn.apache.org/viewvc?rev=1752888&view=rev
Log:
PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/RegisterResolver.java
pig/trunk/test/org/apache/pig/test/TestRegisterParser.java
Modified: pig/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1752888&r1=1752887&r2=1752888&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat Jul 16 05:05:34 2016
@@ -159,6 +159,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4949: Fix registering jar in S3 which was broken by PIG-4417 in Pig 0.16 (yangyishan0901m via daijy)
+
PIG-4821: Pig chararray field with special UTF-8 chars as part of tuple join key produces wrong results in Tez (rohini)
PIG-4734: TOMAP schema inferring breaks some scripts in type checking for bincond (daijy)
Modified: pig/trunk/src/org/apache/pig/parser/RegisterResolver.java
URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/RegisterResolver.java?rev=1752888&r1=1752887&r2=1752888&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/parser/RegisterResolver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/RegisterResolver.java Sat Jul 16 05:05:34 2016
@@ -23,6 +23,10 @@ import java.net.URISyntaxException;
import org.apache.pig.PigServer;
import org.apache.pig.tools.DownloadResolver;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil;
+import org.apache.pig.backend.hadoop.executionengine.shims.HadoopShims;
+import org.apache.hadoop.fs.Path;
public class RegisterResolver {
@@ -66,15 +70,24 @@ public class RegisterResolver {
String scheme = uri.getScheme();
if (scheme != null) {
scheme = scheme.toLowerCase();
+ if (scheme.equals("ivy")) {
+ DownloadResolver downloadResolver = DownloadResolver.getInstance();
+ return downloadResolver.downloadArtifact(uri, pigServer);
+ }
+ if (!hasFileSystemImpl(uri)) {
+ throw new ParserException("Invalid Scheme: " + uri.getScheme());
+ }
}
- if (scheme == null || scheme.equals("file") || scheme.equals("hdfs")) {
- return new URI[] { uri };
- } else if (scheme.equals("ivy")) {
- DownloadResolver downloadResolver = DownloadResolver.getInstance();
- return downloadResolver.downloadArtifact(uri, pigServer);
- } else {
- throw new ParserException("Invalid Scheme: " + uri.getScheme());
- }
+ return new URI[] { uri };
+ }
+
+ /**
+ * @param uri
+ * @return True if the uri has valid file system implementation
+ */
+ private boolean hasFileSystemImpl(URI uri) {
+ Configuration conf = ConfigurationUtil.toConfiguration(pigServer.getPigContext().getProperties(), true);
+ return HadoopShims.hasFileSystemImpl(new Path(uri), conf);
}
/**
Modified: pig/trunk/test/org/apache/pig/test/TestRegisterParser.java
URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestRegisterParser.java?rev=1752888&r1=1752887&r2=1752888&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestRegisterParser.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestRegisterParser.java Sat Jul 16 05:05:34 2016
@@ -30,6 +30,7 @@ import java.io.IOException;
import java.io.Writer;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.Properties;
import org.apache.pig.ExecType;
import org.apache.pig.impl.PigContext;
@@ -42,6 +43,9 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
+import org.apache.pig.impl.util.PropertiesUtil;
+import org.apache.hadoop.fs.LocalFileSystem;
+
public class TestRegisterParser {
private PigServer pigServer;
@@ -49,7 +53,12 @@ public class TestRegisterParser {
@Before
public void setUp() throws Exception {
- pigServer = new PigServer(ExecType.LOCAL);
+ Properties properties = PropertiesUtil.loadDefaultProperties();
+ properties.setProperty("fs.s3.impl", LocalFileSystem.class.getName());
+ properties.setProperty("fs.s3n.impl", LocalFileSystem.class.getName());
+ properties.setProperty("fs.s3a.impl", LocalFileSystem.class.getName());
+
+ pigServer = new PigServer(ExecType.LOCAL, properties);
// Generate test jar files
for (int i = 1; i <= 5; i++) {
@@ -107,6 +116,34 @@ public class TestRegisterParser {
}
}
+ @Test
+ public void testResolveForVariousFileSystemSchemes() throws URISyntaxException, IOException, ParserException {
+ URI[] list = new URI[6];
+ list[0] = new URI("file://test.jar");
+ list[1] = new URI("hdfs://test.jar");
+ list[2] = new URI("s3://test.jar");
+ list[3] = new URI("s3n://test.jar");
+ list[4] = new URI("s3a://test.jar");
+ list[5] = new URI("test.jar");
+
+ RegisterResolver registerResolver = new RegisterResolver(pigServer);
+ for (URI uri : list) {
+ URI[] resolvedUris = registerResolver.resolve(uri);
+ Assert.assertEquals(1, resolvedUris.length);
+ Assert.assertEquals(uri, resolvedUris[0]);
+ }
+ }
+
+ @Test(expected = ParserException.class)
+ public void testResolveParseException() throws URISyntaxException, IOException, ParserException {
+ new RegisterResolver(pigServer).resolve(new URI("abc://test.jar"));
+ }
+
+ @Test(expected = URISyntaxException.class)
+ public void testResolveURISyntaxException() throws URISyntaxException, IOException, ParserException {
+ new RegisterResolver(pigServer).resolve(new URI("123://test.jar"));
+ }
+
// Throw error when a scripting language and namespace is specified for a jar
@Test(expected = ParserException.class)
public void testRegisterJarException1() throws IOException, ParserException {