svn commit: r1562526 - in /hive/trunk/ql/src/java/org/apache/hadoop/hive/ql: Context.java optimizer/GenMapRedUtils.java parse/SemanticAnalyzer.java

[ha...@apache.org]

Author: hashutosh
Date: Wed Jan 29 18:01:47 2014
New Revision: 1562526

URL: http://svn.apache.org/r1562526
Log:
HIVE-6152 : insert query fails on hdfs federation + viewfs (Ashutosh Chauhan via Thejas Nair)

Modified:
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
    hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java?rev=1562526&r1=1562525&r2=1562526&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/Context.java Wed Jan 29 18:01:47 2014
@@ -49,6 +49,7 @@ import org.apache.hadoop.hive.ql.lockmgr
 import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
 import org.apache.hadoop.hive.shims.ShimLoader;
 import org.apache.hadoop.util.StringUtils;
+import org.apache.velocity.runtime.parser.node.GetExecutor;
 
 /**
  * Context for Semantic Analyzers. Usage: not reusable - construct a new one for
@@ -336,6 +337,16 @@ public class Context {
   }
 
   /**
+   * This is similar to getExternalTmpPath() with difference being this method returns temp path
+   * within passed in uri, whereas getExternalTmpPath() ignores passed in path and returns temp
+   * path within /tmp
+   */
+  public Path getExtTmpPathRelTo(URI uri) {
+    return new Path (getScratchDir(uri.getScheme(), uri.getAuthority(), !explain, 
+    uri.getPath() + Path.SEPARATOR + "_" + this.executionId), EXT_PREFIX + nextPathId());
+  }
+
+  /**
    * @return the resFile
    */
   public Path getResFile() {

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java?rev=1562526&r1=1562525&r2=1562526&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMapRedUtils.java Wed Jan 29 18:01:47 2014
@@ -1692,7 +1692,12 @@ public final class GenMapRedUtils {
       // generate the temporary file
       // it must be on the same file system as the current destination
       Context baseCtx = parseCtx.getContext();
-      Path tmpDir = baseCtx.getExternalTmpPath(dest.toUri());
+  	  // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/..
+      // to final location /user/hive/warehouse/ will fail later, so instead pick tmp dir
+      // on same namespace as tbl dir. 
+      Path tmpDir = dest.toUri().getScheme().equals("viewfs") ? 
+        baseCtx.getExtTmpPathRelTo(dest.toUri()) : 
+        baseCtx.getExternalTmpPath(dest.toUri());
 
       FileSinkDesc fileSinkDesc = fsOp.getConf();
       // Change all the linked file sink descriptors

Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1562526&r1=1562525&r2=1562526&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Wed Jan 29 18:01:47 2014
@@ -5333,7 +5333,12 @@ public class SemanticAnalyzer extends Ba
       if (isNonNativeTable) {
         queryTmpdir = dest_path;
       } else {
-        queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri());
+    	// if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/..
+        // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir
+        // on same namespace as tbl dir. 
+        queryTmpdir = dest_path.toUri().getScheme().equals("viewfs") ? 
+          ctx.getExtTmpPathRelTo(dest_path.getParent().toUri()) : 
+          ctx.getExternalTmpPath(dest_path.toUri());
       }
       if (dpCtx != null) {
         // set the root of the temporay path where dynamic partition columns will populate
@@ -5426,7 +5431,12 @@ public class SemanticAnalyzer extends Ba
       dest_path = new Path(tabPath.toUri().getScheme(), tabPath.toUri()
           .getAuthority(), partPath.toUri().getPath());
 
-      queryTmpdir = ctx.getExternalTmpPath(dest_path.toUri());
+      // if we are on viewfs we don't want to use /tmp as tmp dir since rename from /tmp/..
+      // to final /user/hive/warehouse/ will fail later, so instead pick tmp dir
+      // on same namespace as tbl dir. 
+      queryTmpdir = dest_path.toUri().getScheme().equals("viewfs") ? 
+        ctx.getExtTmpPathRelTo(dest_path.getParent().toUri()) : 
+        ctx.getExternalTmpPath(dest_path.toUri());
       table_desc = Utilities.getTableDesc(dest_tab);
 
       // Add sorting/bucketing if needed