You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pig.apache.org by pr...@apache.org on 2010/02/16 19:42:15 UTC
svn commit: r910645 - in /hadoop/pig/branches/load-store-redesign:
CHANGES.txt src/org/apache/pig/LoadFunc.java
test/org/apache/pig/test/TestLoad.java
Author: pradeepkth
Date: Tue Feb 16 18:42:14 2010
New Revision: 910645
URL: http://svn.apache.org/viewvc?rev=910645&view=rev
Log:
PIG-1234: Unable to create input slice for har:// files (pradeepkth)
Modified:
hadoop/pig/branches/load-store-redesign/CHANGES.txt
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java
Modified: hadoop/pig/branches/load-store-redesign/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/CHANGES.txt?rev=910645&r1=910644&r2=910645&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/CHANGES.txt (original)
+++ hadoop/pig/branches/load-store-redesign/CHANGES.txt Tue Feb 16 18:42:14 2010
@@ -22,6 +22,8 @@
INCOMPATIBLE CHANGES
+PIG-1234: Unable to create input slice for har:// files (pradeepkth)
+
PIG-1200: Using TableInputFormat in HBaseStorage (zjffdu via pradeepkth)
PIG-1148: Move splitable logic from pig latin to InputFormat (zjffdu via
Modified: hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java?rev=910645&r1=910644&r2=910645&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java (original)
+++ hadoop/pig/branches/load-store-redesign/src/org/apache/pig/LoadFunc.java Tue Feb 16 18:42:14 2010
@@ -237,21 +237,20 @@
for (String fname: fnames) {
Path p = new Path(fname.trim());
URI uri = p.toUri();
- String scheme = uri.getScheme();
- if (scheme != null) {
- scheme = scheme.toLowerCase();
- }
-
- if (scheme != null && !scheme.equals(fsScheme)) {
- throw new FrontendException("Incompatible file URI scheme: "
- + scheme + " : " + fsScheme);
- }
- String path = uri.getPath();
// if the supplied location has an authority and is absolute, just
// use it
- if(uri.getAuthority() != null && p.isAbsolute()) {
- fname = p.toString();
- } else {
+ if(uri.getAuthority() == null || ! p.isAbsolute()) {
+ String scheme = uri.getScheme();
+ if (scheme != null) {
+ scheme = scheme.toLowerCase();
+ }
+
+ if (scheme != null && !scheme.equals(fsScheme)) {
+ throw new FrontendException("Incompatible file URI scheme: "
+ + scheme + " : " + fsScheme);
+ }
+ String path = uri.getPath();
+
fname = (p.isAbsolute()) ?
new Path(rootDir, path).toString() :
new Path(curDir, path).toString();
Modified: hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java
URL: http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java?rev=910645&r1=910644&r2=910645&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java (original)
+++ hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java Tue Feb 16 18:42:14 2010
@@ -51,6 +51,7 @@
import org.apache.pig.impl.logicalLayer.LogicalPlanBuilder;
import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.test.utils.GenPhyOp;
+import org.apache.pig.test.utils.LogicalPlanTester;
import org.apache.pig.test.utils.TestHelper;
import org.junit.After;
import org.junit.Before;
@@ -63,6 +64,7 @@
static MiniCluster cluster = MiniCluster.buildCluster();
+ @Override
@Before
public void setUp() throws Exception {
FileLocalizer.deleteTempFiles();
@@ -72,6 +74,7 @@
};
}
+ @Override
@After
public void tearDown() throws Exception {
}
@@ -207,7 +210,16 @@
pc = pig.getPigContext();
checkLoadPath("usr/pig/{a,c},/usr/pig/b","/tmp/usr/pig/{a,c},/usr/pig/b");
}
- }
+ }
+
+ @Test
+ public void testNonDfsLocation() throws Exception {
+ LogicalPlanTester lpt = new LogicalPlanTester();
+ String nonDfsUrl = "har://hdfs-namenode/user/foo/";
+ LogicalPlan lp = lpt.buildPlan("a = load '" + nonDfsUrl + "';");
+ LOLoad load = (LOLoad) lp.getRoots().get(0);
+ Assert.assertEquals(nonDfsUrl, load.getInputFile().getFileName());
+ }
private void checkLoadPath(String orig, String expected) throws Exception {