You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2017/08/03 07:32:38 UTC
hive git commit: HIVE-17144: export of temporary tables not working
and it seems to be using distcp rather than filesystem copy (Anishek Agarwal,
reviewed by Daniel Dai)
Repository: hive
Updated Branches:
refs/heads/master 8ff37ad87 -> 68b2f9b2a
HIVE-17144: export of temporary tables not working and it seems to be using distcp rather than filesystem copy (Anishek Agarwal, reviewed by Daniel Dai)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/68b2f9b2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/68b2f9b2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/68b2f9b2
Branch: refs/heads/master
Commit: 68b2f9b2a78d9727f034ece8b3d7adc1c29054ad
Parents: 8ff37ad
Author: Daniel Dai <da...@hortonworks.com>
Authored: Thu Aug 3 00:32:29 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu Aug 3 00:32:29 2017 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/WarehouseInstance.java | 17 +++++++++++------
.../hive/ql/parse/repl/dump/TableExport.java | 2 +-
.../hive/ql/parse/repl/dump/io/FileOperations.java | 4 +---
3 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/68b2f9b2/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index 1128eae..8dfab08 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -64,7 +64,7 @@ class WarehouseInstance implements Closeable {
private final static String LISTENER_CLASS = DbNotificationListener.class.getCanonicalName();
- WarehouseInstance(Logger logger, MiniDFSCluster cluster) throws Exception {
+ WarehouseInstance(Logger logger, MiniDFSCluster cluster, boolean hiveInTests) throws Exception {
this.logger = logger;
this.miniDFSCluster = cluster;
assert miniDFSCluster.isClusterUp();
@@ -73,10 +73,14 @@ class WarehouseInstance implements Closeable {
Path cmRootPath = mkDir(fs, "/cmroot" + uniqueIdentifier);
this.functionsRoot = mkDir(fs, "/functions" + uniqueIdentifier).toString();
- initialize(cmRootPath.toString());
+ initialize(cmRootPath.toString(), hiveInTests);
+ }
+
+ WarehouseInstance(Logger logger, MiniDFSCluster cluster) throws Exception {
+ this(logger, cluster, true);
}
- private void initialize(String cmRoot) throws Exception {
+ private void initialize(String cmRoot, boolean hiveInTest) throws Exception {
hiveConf = new HiveConf(miniDFSCluster.getConfiguration(0), TestReplicationScenarios.class);
String metaStoreUri = System.getProperty("test." + HiveConf.ConfVars.METASTOREURIS.varname);
String hiveWarehouseLocation = System.getProperty("test.warehouse.dir", "/tmp")
@@ -89,6 +93,7 @@ class WarehouseInstance implements Closeable {
return;
}
+ hiveConf.setBoolVar(HiveConf.ConfVars.HIVE_IN_TEST, hiveInTest);
// turn on db notification listener on meta store
hiveConf.setVar(HiveConf.ConfVars.METASTORE_TRANSACTIONAL_EVENT_LISTENERS, LISTENER_CLASS);
hiveConf.setBoolVar(HiveConf.ConfVars.REPLCMENABLED, true);
@@ -177,8 +182,7 @@ class WarehouseInstance implements Closeable {
}
WarehouseInstance verify(String data) throws IOException {
- verifyResults(data == null ? new String[] {} : new String[] { data });
- return this;
+ return verifyResults(data == null ? new String[] {} : new String[] { data });
}
/**
@@ -188,7 +192,7 @@ class WarehouseInstance implements Closeable {
* Unless for Null Values it actually returns in UpperCase and hence explicitly lowering case
* before assert.
*/
- private void verifyResults(String[] data) throws IOException {
+ WarehouseInstance verifyResults(String[] data) throws IOException {
List<String> results = getOutput();
logger.info("Expecting {}", StringUtils.join(data, ","));
logger.info("Got {}", results);
@@ -196,6 +200,7 @@ class WarehouseInstance implements Closeable {
for (int i = 0; i < data.length; i++) {
assertEquals(data[i].toLowerCase(), results.get(i).toLowerCase());
}
+ return this;
}
List<String> getOutput() throws IOException {
http://git-wip-us.apache.org/repos/asf/hive/blob/68b2f9b2/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
index 9f22f23..4d23efc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/TableExport.java
@@ -57,7 +57,7 @@ public class TableExport {
throws SemanticException {
this.tableSpec = (tableSpec != null
&& tableSpec.tableHandle.isTemporary()
- && !replicationSpec.isInReplicationScope())
+ && replicationSpec.isInReplicationScope())
? null
: tableSpec;
this.replicationSpec = replicationSpec;
http://git-wip-us.apache.org/repos/asf/hive/blob/68b2f9b2/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
index 164ca74..e1e3ae1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/io/FileOperations.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.ReplChangeManager;
-import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.LoadSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
@@ -70,8 +69,7 @@ public class FileOperations {
for (FileStatus fileStatus : fileStatuses) {
srcPaths.add(fileStatus.getPath());
}
-
- ReplCopyTask.doCopy(exportRootDataDir, exportFileSystem, srcPaths, dataFileSystem, hiveConf);
+ new CopyUtils(hiveConf).doCopy(exportRootDataDir, srcPaths);
}
/**