You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2018/02/08 08:08:45 UTC
hive git commit: HIVE-18626: Repl load "with" clause does not pass
config to tasks (Daniel Dai, reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master 16b8575b7 -> 637224fcc
HIVE-18626: Repl load "with" clause does not pass config to tasks (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/637224fc
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/637224fc
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/637224fc
Branch: refs/heads/master
Commit: 637224fcce031bb036718621d05bb714902562a0
Parents: 16b8575
Author: Daniel Dai <da...@hortonworks.com>
Authored: Thu Feb 8 00:08:32 2018 -0800
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu Feb 8 00:08:32 2018 -0800
----------------------------------------------------------------------
.../hive/ql/exec/repl/bootstrap/load/LoadDatabase.java | 6 +++---
.../ql/exec/repl/bootstrap/load/table/LoadPartitions.java | 10 ++++++----
.../hive/ql/exec/repl/bootstrap/load/table/LoadTable.java | 2 +-
.../hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java | 4 ++--
.../org/apache/hadoop/hive/ql/plan/ImportTableDesc.java | 8 ++++----
5 files changed, 16 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java
index 537c5e9..9191d22 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java
@@ -91,14 +91,14 @@ public class LoadDatabase {
// db.
// TODO: we might revisit this in create-drop-recreate cases, needs some thinking on.
DDLWork work = new DDLWork(new HashSet<>(), new HashSet<>(), createDbDesc);
- return TaskFactory.get(work, context.hiveConf);
+ return TaskFactory.get(work, context.hiveConf, true);
}
private static Task<? extends Serializable> alterDbTask(Database dbObj, HiveConf hiveConf) {
AlterDatabaseDesc alterDbDesc =
new AlterDatabaseDesc(dbObj.getName(), dbObj.getParameters(), null);
DDLWork work = new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc);
- return TaskFactory.get(work, hiveConf);
+ return TaskFactory.get(work, hiveConf, true);
}
private Task<? extends Serializable> setOwnerInfoTask(Database dbObj) {
@@ -106,7 +106,7 @@ public class LoadDatabase {
new PrincipalDesc(dbObj.getOwnerName(), dbObj.getOwnerType()),
null);
DDLWork work = new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc);
- return TaskFactory.get(work, context.hiveConf);
+ return TaskFactory.get(work, context.hiveConf, true);
}
private boolean existEmptyDb(String dbName) throws InvalidOperationException, HiveException {
http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
index bad7962..0a82225 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadPartitions.java
@@ -113,7 +113,7 @@ public class LoadPartitions {
private void createTableReplLogTask() throws SemanticException {
ReplStateLogWork replLogWork = new ReplStateLogWork(replLogger,
tableDesc.getTableName(), tableDesc.tableType());
- Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf);
+ Task<ReplStateLogWork> replLogTask = TaskFactory.get(replLogWork, context.hiveConf, true);
if (tracker.tasks().isEmpty()) {
tracker.addTask(replLogTask);
@@ -224,7 +224,8 @@ public class LoadPartitions {
Task<?> addPartTask = TaskFactory.get(
new DDLWork(new HashSet<>(), new HashSet<>(), addPartitionDesc),
- context.hiveConf
+ context.hiveConf,
+ true
);
Task<?> movePartitionTask = movePartitionTask(table, partSpec, tmpPath);
@@ -246,7 +247,7 @@ public class LoadPartitions {
);
loadTableWork.setInheritTableSpecs(false);
MoveWork work = new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
- return TaskFactory.get(work, context.hiveConf);
+ return TaskFactory.get(work, context.hiveConf, true);
}
private Path locationOnReplicaWarehouse(Table table, AddPartitionDesc.OnePartitionDesc partSpec)
@@ -274,7 +275,8 @@ public class LoadPartitions {
desc.getPartition(0).setLocation(ptn.getLocation()); // use existing location
return TaskFactory.get(
new DDLWork(new HashSet<>(), new HashSet<>(), desc),
- context.hiveConf
+ context.hiveConf,
+ true
);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
index 3b8cb68..e0721f1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/table/LoadTable.java
@@ -234,7 +234,7 @@ public class LoadTable {
);
MoveWork moveWork =
new MoveWork(new HashSet<>(), new HashSet<>(), loadTableWork, null, false);
- Task<?> loadTableTask = TaskFactory.get(moveWork, context.hiveConf);
+ Task<?> loadTableTask = TaskFactory.get(moveWork, context.hiveConf, true);
copyTask.addDependentTask(loadTableTask);
return copyTask;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index 1d89661..c38f7dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -469,7 +469,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
alterTblDesc.setPartSpec((HashMap<String, String>)partSpec);
Task<? extends Serializable> updateReplIdTask = TaskFactory.get(
- new DDLWork(inputs, outputs, alterTblDesc), conf);
+ new DDLWork(inputs, outputs, alterTblDesc), conf, true);
// Link the update repl state task with dependency collection task
if (preCursor != null) {
@@ -491,7 +491,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
AlterDatabaseDesc alterDbDesc = new AlterDatabaseDesc(
dbName, mapProp, new ReplicationSpec(replState, replState));
Task<? extends Serializable> updateReplIdTask = TaskFactory.get(
- new DDLWork(inputs, outputs, alterDbDesc), conf);
+ new DDLWork(inputs, outputs, alterDbDesc), conf, true);
// Link the update repl state task with dependency collection task
if (preCursor != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/637224fc/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
index bdfb632..aef83b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ImportTableDesc.java
@@ -317,10 +317,10 @@ public class ImportTableDesc {
public Task<? extends Serializable> getCreateTableTask(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
HiveConf conf) {
switch (getDescType()) {
- case TABLE:
- return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf);
- case VIEW:
- return TaskFactory.get(new DDLWork(inputs, outputs, createViewDesc), conf);
+ case TABLE:
+ return TaskFactory.get(new DDLWork(inputs, outputs, createTblDesc), conf, true);
+ case VIEW:
+ return TaskFactory.get(new DDLWork(inputs, outputs, createViewDesc), conf, true);
}
return null;
}