[hive] branch master updated: HIVE-21186: External tables replication throws NPE if hive.repl.replica.external.table.base.dir is not fully qualified HDFS path (Sankar Hariappan, reviewed by Mahesh Kumar Behera)

[sa...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git


The following commit(s) were added to refs/heads/master by this push:
     new 4271bbf  HIVE-21186: External tables replication throws NPE if hive.repl.replica.external.table.base.dir is not fully qualified HDFS path (Sankar Hariappan, reviewed by Mahesh Kumar Behera)
4271bbf is described below

commit 4271bbfb02ae81901b1b44e15e0ca6bd1407de9d
Author: Sankar Hariappan <sa...@apache.org>
AuthorDate: Thu Jan 31 09:41:01 2019 +0530

    HIVE-21186: External tables replication throws NPE if hive.repl.replica.external.table.base.dir is not fully qualified HDFS path (Sankar Hariappan, reviewed by Mahesh Kumar Behera)
    
    Signed-off-by: Sankar Hariappan <sa...@apache.org>
---
 .../TestReplicationScenariosExternalTables.java    |  8 +++++++-
 .../hive/ql/exec/repl/ReplExternalTables.java      | 22 +++++++++++++++-------
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
index 0e3cefc..40ce4b4 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosExternalTables.java
@@ -216,7 +216,13 @@ public class TestReplicationScenariosExternalTables extends BaseReplicationAcros
     DistributedFileSystem fs = primary.miniDFSCluster.getFileSystem();
     fs.mkdirs(externalTableLocation, new FsPermission("777"));
 
-    List<String> loadWithClause = externalTableBasePathWithClause();
+    // Create base directory but use HDFS path without schema or authority details.
+    // Hive should pick up the local cluster's HDFS schema/authority.
+    externalTableBasePathWithClause();
+    List<String> loadWithClause = Collections.singletonList(
+            "'" + HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname + "'='"
+                    + REPLICA_EXTERNAL_BASE + "'"
+    );
 
     WarehouseInstance.Tuple bootstrapTuple = primary.run("use " + primaryDbName)
         .run("create external table a (i int, j int) "
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java
index 012df9d..59b7c1c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplExternalTables.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.metastore.TableType;
+import org.apache.hadoop.hive.ql.ErrorMsg;
 import org.apache.hadoop.hive.ql.metadata.Hive;
 import org.apache.hadoop.hive.ql.metadata.HiveException;
 import org.apache.hadoop.hive.ql.metadata.Partition;
@@ -38,7 +39,6 @@ import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.OutputStream;
 import java.io.StringWriter;
-import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.util.Base64;
 import java.util.HashSet;
@@ -62,13 +62,21 @@ public final class ReplExternalTables {
 
   private ReplExternalTables(){}
 
-  public static String externalTableLocation(HiveConf hiveConf, String location) {
-    String currentPath = new Path(location).toUri().getPath();
+  public static String externalTableLocation(HiveConf hiveConf, String location) throws SemanticException {
     String baseDir = hiveConf.get(HiveConf.ConfVars.REPL_EXTERNAL_TABLE_BASE_DIR.varname);
-    URI basePath = new Path(baseDir).toUri();
-    String dataPath = currentPath.replaceFirst(Path.SEPARATOR, basePath.getPath() + Path.SEPARATOR);
-    Path dataLocation = new Path(basePath.getScheme(), basePath.getAuthority(), dataPath);
-    LOG.debug("incoming location: {} , new location: {}", location, dataLocation.toString());
+    Path basePath = new Path(baseDir);
+    Path currentPath = new Path(location);
+    String targetPathWithoutSchemeAndAuth = basePath.toUri().getPath() + currentPath.toUri().getPath();
+    Path dataLocation;
+    try {
+      dataLocation = PathBuilder.fullyQualifiedHDFSUri(
+              new Path(targetPathWithoutSchemeAndAuth),
+              basePath.getFileSystem(hiveConf)
+      );
+    } catch (IOException e) {
+      throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e);
+    }
+    LOG.info("Incoming external table location: {} , new location: {}", location, dataLocation.toString());
     return dataLocation.toString();
   }