You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by da...@apache.org on 2010/05/17 03:48:24 UTC
svn commit: r944950 - in /hadoop/pig/branches/branch-0.7:
src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
test/org/apache/pig/test/TestParser.java
Author: daijy
Date: Mon May 17 01:48:24 2010
New Revision: 944950
URL: http://svn.apache.org/viewvc?rev=944950&view=rev
Log:
PIG-1403: Make Pig work with remote HDFS in secure mode
Modified:
hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java
Modified: hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=944950&r1=944949&r2=944950&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt (original)
+++ hadoop/pig/branches/branch-0.7/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt Mon May 17 01:48:24 2010
@@ -1381,35 +1381,37 @@ LogicalOperator LoadClause(LogicalPlan l
if (absolutePath == null) {
absolutePath = loFunc.relativeToAbsolutePath(filename, getCurrentDir(pigContext));
- // Get native host
- String defaultFS = (String)pigContext.getProperties().get("fs.default.name");
- URI defaultFSURI = new URI(defaultFS);
- String defaultHost = defaultFSURI.getHost();
- if (defaultHost==null)
- defaultHost="";
- defaultHost = defaultHost.toLowerCase();
-
- Set<String> remoteHosts = getRemoteHosts(absolutePath, defaultHost);
-
- String hdfsServersString = (String)pigContext.getProperties().get("mapreduce.job.hdfs-servers");
- if (hdfsServersString==null) hdfsServersString="";
- String hdfsServers[] = hdfsServersString.split(",");
-
- for (String remoteHost : remoteHosts) {
- boolean existing = false;
- for (String hdfsServer:hdfsServers) {
- if (hdfsServer.equals(remoteHost))
- existing = true;
- }
- if (!existing) {
- if (!hdfsServersString.isEmpty())
- hdfsServersString = hdfsServersString + ",";
- hdfsServersString = hdfsServersString + remoteHost;
- }
- }
-
- if (!hdfsServersString.isEmpty())
- pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", hdfsServersString);
+ if (absolutePath!=null) {
+ // Get native host
+ String defaultFS = (String)pigContext.getProperties().get("fs.default.name");
+ URI defaultFSURI = new URI(defaultFS);
+ String defaultHost = defaultFSURI.getHost();
+ if (defaultHost==null)
+ defaultHost="";
+ defaultHost = defaultHost.toLowerCase();
+
+ Set<String> remoteHosts = getRemoteHosts(absolutePath, defaultHost);
+
+ String hdfsServersString = (String)pigContext.getProperties().get("mapreduce.job.hdfs-servers");
+ if (hdfsServersString==null) hdfsServersString="";
+ String hdfsServers[] = hdfsServersString.split(",");
+
+ for (String remoteHost : remoteHosts) {
+ boolean existing = false;
+ for (String hdfsServer:hdfsServers) {
+ if (hdfsServer.equals(remoteHost))
+ existing = true;
+ }
+ if (!existing) {
+ if (!hdfsServersString.isEmpty())
+ hdfsServersString = hdfsServersString + ",";
+ hdfsServersString = hdfsServersString + remoteHost;
+ }
+ }
+
+ if (!hdfsServersString.isEmpty())
+ pigContext.getProperties().setProperty("mapreduce.job.hdfs-servers", hdfsServersString);
+ }
fileNameMap.put(constructFileNameSignature(filename, funcSpec), absolutePath);
}
lo = new LOLoad(lp, new OperatorKey(scope, getNextId()), new FileSpec(absolutePath, funcSpec),
Modified: hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java?rev=944950&r1=944949&r2=944950&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java (original)
+++ hadoop/pig/branches/branch-0.7/test/org/apache/pig/test/TestParser.java Mon May 17 01:48:24 2010
@@ -92,30 +92,22 @@ protected final Log log = LogFactory.get
public void testRemoteServerList() throws ExecException, IOException {
try {
Properties pigProperties = pigServer.getPigContext().getProperties();
- pigProperties.setProperty("fs.default.name", "hdfs://a.com:8020");
-
+
pigServer.registerQuery("a = load '/user/pig/1.txt';");
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null);
- pigServer.registerQuery("a = load 'hdfs://a.com/user/pig/1.txt';");
- assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null);
-
- pigServer.registerQuery("a = load 'har:///1.txt';");
- assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")==null);
-
pigServer.registerQuery("a = load 'hdfs://b.com/user/pig/1.txt';");
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null &&
pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://b.com"));
- pigServer.registerQuery("a = load 'har://hdfs-c.com/user/pig/1.txt';");
+ pigServer.registerQuery("a = load 'har://hdfs-c.com:8020/user/pig/1.txt';");
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null &&
- pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://c.com"));
+ pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://c.com:8020"));
pigServer.registerQuery("a = load 'hdfs://d.com:8020/user/pig/1.txt';");
assertTrue(pigProperties.getProperty("mapreduce.job.hdfs-servers")!=null &&
pigProperties.getProperty("mapreduce.job.hdfs-servers").contains("hdfs://d.com:8020"));
-
} catch (IOException io) {
}
}