You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2018/03/28 16:51:14 UTC
hive git commit: HIVE-19007: Support REPL LOAD from primary using
replica connection configurations received through WITH clause (Sankar
Hariappan, reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master 9b84ed4b2 -> 6edff8a91
HIVE-19007: Support REPL LOAD from primary using replica connection configurations received through WITH clause (Sankar Hariappan, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6edff8a9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6edff8a9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6edff8a9
Branch: refs/heads/master
Commit: 6edff8a91fa4db1ba0450f81fdf1c1c1beec50d5
Parents: 9b84ed4
Author: Sankar Hariappan <sa...@apache.org>
Authored: Wed Mar 28 22:20:29 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Wed Mar 28 22:20:29 2018 +0530
----------------------------------------------------------------------
...TestReplicationScenariosAcrossInstances.java | 96 ++++++++++++++++++++
.../hadoop/hive/ql/parse/WarehouseInstance.java | 23 +++++
.../hive/ql/parse/ImportSemanticAnalyzer.java | 36 ++++----
.../ql/parse/ReplicationSemanticAnalyzer.java | 36 +++++---
.../repl/load/message/AddForeignKeyHandler.java | 3 +-
.../message/AddNotNullConstraintHandler.java | 3 +-
.../repl/load/message/AddPrimaryKeyHandler.java | 3 +-
.../message/AddUniqueConstraintHandler.java | 3 +-
.../repl/load/message/AlterDatabaseHandler.java | 2 +-
.../load/message/CreateDatabaseHandler.java | 6 +-
.../load/message/CreateFunctionHandler.java | 4 +-
.../load/message/DropConstraintHandler.java | 3 +-
.../repl/load/message/DropDatabaseHandler.java | 2 +-
.../repl/load/message/DropFunctionHandler.java | 2 +-
.../repl/load/message/DropPartitionHandler.java | 2 +-
.../repl/load/message/DropTableHandler.java | 2 +-
.../load/message/RenamePartitionHandler.java | 2 +-
.../repl/load/message/RenameTableHandler.java | 2 +-
.../load/message/TruncatePartitionHandler.java | 2 +-
.../repl/load/message/TruncateTableHandler.java | 2 +-
20 files changed, 181 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 13b918d..689ca76 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -524,4 +524,100 @@ public class TestReplicationScenariosAcrossInstances {
*/
}
+
+ @Test
+ public void testReplLoadFromSourceUsingWithClause() throws Throwable {
+ HiveConf replicaConf = replica.getConf();
+ List<String> withConfigs = Arrays.asList(
+ "'hive.metastore.warehouse.dir'='" + replicaConf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE) + "'",
+ "'hive.metastore.uris'='" + replicaConf.getVar(HiveConf.ConfVars.METASTOREURIS) + "'",
+ "'hive.repl.replica.functions.root.dir'='" + replicaConf.getVar(HiveConf.ConfVars.REPL_FUNCTIONS_ROOT_DIR) + "'");
+
+ //////////// Bootstrap ////////////
+ WarehouseInstance.Tuple bootstrapTuple = primary
+ .run("use " + primaryDbName)
+ .run("create table table1 (i int)")
+ .run("create table table2 (id int) partitioned by (country string)")
+ .run("insert into table1 values (1)")
+ .dump(primaryDbName, null);
+
+ // Run load on primary itself
+ primary.load(replicatedDbName, bootstrapTuple.dumpLocation, withConfigs)
+ .status(replicatedDbName, withConfigs)
+ .verifyResult(bootstrapTuple.lastReplicationId);
+
+ replica.run("use " + replicatedDbName)
+ .run("show tables")
+ .verifyResults(new String[] { "table1", "table2" })
+ .run("select * from table1")
+ .verifyResults(new String[]{ "1" });
+
+ //////////// First Incremental ////////////
+ WarehouseInstance.Tuple incrementalOneTuple = primary
+ .run("use " + primaryDbName)
+ .run("alter table table1 rename to renamed_table1")
+ .run("insert into table2 partition(country='india') values (1) ")
+ .run("insert into table2 partition(country='usa') values (2) ")
+ .run("create table table3 (i int)")
+ .run("insert into table3 values(10)")
+ .run("create function " + primaryDbName
+ + ".testFunctionOne as 'hivemall.tools.string.StopwordUDF' "
+ + "using jar 'ivy://io.github.myui:hivemall:0.4.0-2'")
+ .dump(primaryDbName, bootstrapTuple.lastReplicationId);
+
+ // Run load on primary itself
+ primary.load(replicatedDbName, incrementalOneTuple.dumpLocation, withConfigs)
+ .status(replicatedDbName, withConfigs)
+ .verifyResult(incrementalOneTuple.lastReplicationId);
+
+ replica.run("use " + replicatedDbName)
+ .run("show tables")
+ .verifyResults(new String[] { "renamed_table1", "table2", "table3" })
+ .run("select * from renamed_table1")
+ .verifyResults(new String[] { "1" })
+ .run("select id from table2 order by id")
+ .verifyResults(new String[] { "1", "2" })
+ .run("select * from table3")
+ .verifyResults(new String[] { "10" })
+ .run("show functions like '" + replicatedDbName + "*'")
+ .verifyResult(replicatedDbName + ".testFunctionOne");
+
+ //////////// Second Incremental ////////////
+ WarehouseInstance.Tuple secondIncremental = primary
+ .run("use " + primaryDbName)
+ .run("alter table table2 add columns (zipcode int)")
+ .run("alter table table3 set tblproperties('custom.property'='custom.value')")
+ .run("drop table renamed_table1")
+ .run("alter table table2 drop partition(country='usa')")
+ .run("truncate table table3")
+ .run("drop function " + primaryDbName + ".testFunctionOne ")
+ .dump(primaryDbName, incrementalOneTuple.lastReplicationId);
+
+ // Run load on primary itself
+ primary.load(replicatedDbName, secondIncremental.dumpLocation, withConfigs)
+ .status(replicatedDbName, withConfigs)
+ .verifyResult(secondIncremental.lastReplicationId);
+
+ replica.run("use " + replicatedDbName)
+ .run("show tables")
+ .verifyResults(new String[] { "table2", "table3"})
+ .run("desc table2")
+ .verifyResults(new String[] {
+ "id \tint \t ",
+ "country \tstring \t ",
+ "zipcode \tint \t ",
+ "\t \t ",
+ "# Partition Information\t \t ",
+ "# col_name \tdata_type \tcomment ",
+ "country \tstring \t ",
+ })
+ .run("show tblproperties table3('custom.property')")
+ .verifyResults(new String[] { "custom.value\t " })
+ .run("select id from table2 order by id")
+ .verifyResults(new String[] { "1" })
+ .run("select * from table3")
+ .verifyResults(Collections.emptyList())
+ .run("show functions like '" + replicatedDbName + "*'")
+ .verifyResult(null);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index bc70fe4..0006ea5 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -146,6 +146,10 @@ public class WarehouseInstance implements Closeable {
return PathBuilder.fullyQualifiedHDFSUri(path, fs);
}
+ public HiveConf getConf() {
+ return hiveConf;
+ }
+
private int next = 0;
private void advanceDumpDir() {
@@ -201,6 +205,25 @@ public class WarehouseInstance implements Closeable {
return this;
}
+ WarehouseInstance load(String replicatedDbName, String dumpLocation, List<String> withClauseOptions)
+ throws Throwable {
+ String replLoadCmd = "REPL LOAD " + replicatedDbName + " FROM '" + dumpLocation + "'";
+ if (!withClauseOptions.isEmpty()) {
+ replLoadCmd += " WITH (" + StringUtils.join(withClauseOptions, ",") + ")";
+ }
+ run("EXPLAIN " + replLoadCmd);
+ printOutput();
+ return run(replLoadCmd);
+ }
+
+ WarehouseInstance status(String replicatedDbName, List<String> withClauseOptions) throws Throwable {
+ String replStatusCmd = "REPL STATUS " + replicatedDbName;
+ if (!withClauseOptions.isEmpty()) {
+ replStatusCmd += " WITH (" + StringUtils.join(withClauseOptions, ",") + ")";
+ }
+ return run(replStatusCmd);
+ }
+
WarehouseInstance verifyResult(String data) throws IOException {
verifyResults(data == null ? new String[] {} : new String[] { data });
return this;
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 9a91e3f..e862589 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -424,7 +424,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
loadPath, Utilities.getTableDesc(table), new TreeMap<>(), lft, writeId);
loadTableWork.setStmtId(stmtId);
MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false);
- Task<?> loadTableTask = TaskFactory.get(mv);
+ Task<?> loadTableTask = TaskFactory.get(mv, x.getConf());
copyTask.addDependentTask(loadTableTask);
x.getTasks().add(copyTask);
return loadTableTask;
@@ -466,11 +466,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
} else {
partSpec.setLocation(ptn.getLocation()); // use existing location
}
- return TaskFactory.get(new DDLWork(
- x.getInputs(),
- x.getOutputs(),
- addPartitionDesc
- ));
+ return TaskFactory.get(new DDLWork(x.getInputs(), x.getOutputs(), addPartitionDesc), x.getConf());
}
private static Task<?> addSinglePartition(URI fromURI, FileSystem fs, ImportTableDesc tblDesc,
@@ -484,8 +480,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
+ partSpecToString(partSpec.getPartSpec()));
// addPartitionDesc already has the right partition location
@SuppressWarnings("unchecked")
- Task<?> addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
- x.getOutputs(), addPartitionDesc));
+ Task<?> addPartTask = TaskFactory.get(
+ new DDLWork(x.getInputs(), x.getOutputs(), addPartitionDesc), x.getConf());
return addPartTask;
} else {
String srcLocation = partSpec.getLocation();
@@ -518,8 +514,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
copyTask = TaskFactory.get(cw);
}
- Task<?> addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
- x.getOutputs(), addPartitionDesc));
+ Task<?> addPartTask = TaskFactory.get(
+ new DDLWork(x.getInputs(), x.getOutputs(), addPartitionDesc), x.getConf());
+
// Note: this sets LoadFileType incorrectly for ACID; is that relevant for import?
// See setLoadFileType and setIsAcidIow calls elsewhere for an example.
LoadTableDesc loadTableWork = new LoadTableDesc(moveTaskSrc, Utilities.getTableDesc(table),
@@ -528,8 +525,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
writeId);
loadTableWork.setStmtId(stmtId);
loadTableWork.setInheritTableSpecs(false);
- Task<?> loadPartTask = TaskFactory.get(new MoveWork(
- x.getInputs(), x.getOutputs(), loadTableWork, null, false));
+ Task<?> loadPartTask = TaskFactory.get(
+ new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false),
+ x.getConf());
copyTask.addDependentTask(loadPartTask);
addPartTask.addDependentTask(loadPartTask);
x.getTasks().add(copyTask);
@@ -830,7 +828,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
if (table != null) {
if (table.isPartitioned()) {
x.getLOG().debug("table partitioned");
- Task<?> ict = createImportCommitTask(
+ Task<?> ict = createImportCommitTask(x.getConf(),
table.getDbName(), table.getTableName(), writeId, stmtId,
AcidUtils.isInsertOnlyTable(table.getParameters()));
@@ -867,7 +865,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
x.getOutputs().add(new WriteEntity(parentDb, WriteEntity.WriteType.DDL_SHARED));
if (isPartitioned(tblDesc)) {
- Task<?> ict = createImportCommitTask(
+ Task<?> ict = createImportCommitTask(x.getConf(),
tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
for (AddPartitionDesc addPartitionDesc : partitionDescs) {
@@ -903,10 +901,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
}
private static Task<?> createImportCommitTask(
- String dbName, String tblName, Long writeId, int stmtId, boolean isMmTable) {
+ HiveConf conf, String dbName, String tblName, Long writeId, int stmtId, boolean isMmTable) {
// TODO: noop, remove?
Task<ImportCommitWork> ict = (!isMmTable) ? null : TaskFactory.get(
- new ImportCommitWork(dbName, tblName, writeId, stmtId));
+ new ImportCommitWork(dbName, tblName, writeId, stmtId), conf);
return ict;
}
@@ -994,7 +992,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
if (!replicationSpec.isMetadataOnly()) {
if (isPartitioned(tblDesc)) {
- Task<?> ict = createImportCommitTask(
+ Task<?> ict = createImportCommitTask(x.getConf(),
tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
for (AddPartitionDesc addPartitionDesc : partitionDescs) {
@@ -1021,8 +1019,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
org.apache.hadoop.hive.ql.metadata.Partition ptn = null;
Task<?> ict = replicationSpec.isMetadataOnly() ? null : createImportCommitTask(
- tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
- AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
+ x.getConf(), tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
+ AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
if ((ptn = x.getHive().getPartition(table, partSpec, false)) == null) {
if (!replicationSpec.isMetadataOnly()){
x.getTasks().add(addSinglePartition(
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index 753f039..9f18efd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -82,8 +82,9 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
// of any other queries running in the session
private HiveConf conf;
- // This will be set to true only if repl-status is fired with "WITH" keyword
- private boolean needNewdb;
+ // By default, this will be same as that of super class BaseSemanticAnalyzer. But need to obtain again
+ // if the Hive configs are received from WITH clause in REPL LOAD or REPL STATUS commands.
+ private Hive db;
private static String testInjectDumpDir = null; // unit tests can overwrite this to affect default dump behaviour
private static final String dumpSchema = "dump_dir,last_repl_id#string,string";
@@ -93,6 +94,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
ReplicationSemanticAnalyzer(QueryState queryState) throws SemanticException {
super(queryState);
+ this.db = super.db;
this.conf = new HiveConf(super.conf);
}
@@ -226,6 +228,13 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
for (Map.Entry<String, String> config : replConfigs.entrySet()) {
conf.set(config.getKey(), config.getValue());
}
+
+ // As hive conf is changed, need to get the Hive DB again with it.
+ try {
+ db = Hive.get(conf);
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
}
break;
default:
@@ -420,7 +429,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
Map<String, String> dbProps = new HashMap<>();
dbProps.put(ReplicationSpec.KEY.CURR_STATE_ID.toString(), String.valueOf(dmd.getEventTo()));
ReplStateLogWork replStateLogWork = new ReplStateLogWork(replLogger, dbProps);
- Task<? extends Serializable> barrierTask = TaskFactory.get(replStateLogWork);
+ Task<? extends Serializable> barrierTask = TaskFactory.get(replStateLogWork, conf);
taskChainTail.addDependentTask(barrierTask);
LOG.debug("Added {}:{} as a precursor of barrier task {}:{}",
taskChainTail.getClass(), taskChainTail.getId(),
@@ -560,7 +569,6 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
// REPL STATUS
private void initReplStatus(ASTNode ast) throws SemanticException{
- needNewdb = false;
dbNameOrPattern = PlanUtils.stripQuotes(ast.getChild(0).getText());
int numChildren = ast.getChildCount();
for (int i = 1; i < numChildren; i++) {
@@ -576,8 +584,14 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
for (Map.Entry<String, String> config : replConfigs.entrySet()) {
conf.set(config.getKey(), config.getValue());
}
+
+ // As hive conf is changed, need to get the Hive DB again with it.
+ try {
+ db = Hive.get(conf);
+ } catch (HiveException e) {
+ throw new SemanticException(e);
+ }
}
- needNewdb = true;
break;
default:
throw new SemanticException("Unrecognized token in REPL STATUS statement");
@@ -592,16 +606,9 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
String replLastId = null;
try {
- Hive newDb;
- if (needNewdb) {
- newDb = Hive.get(conf, false);
- } else {
- newDb = db;
- }
-
if (tblNameOrPattern != null) {
// Checking for status of table
- Table tbl = newDb.getTable(dbNameOrPattern, tblNameOrPattern);
+ Table tbl = db.getTable(dbNameOrPattern, tblNameOrPattern);
if (tbl != null) {
inputs.add(new ReadEntity(tbl));
Map<String, String> params = tbl.getParameters();
@@ -611,8 +618,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
}
} else {
// Checking for status of a db
-
- Database database = newDb.getDatabase(dbNameOrPattern);
+ Database database = db.getDatabase(dbNameOrPattern);
if (database != null) {
inputs.add(new ReadEntity(database));
Map<String, String> params = database.getParameters();
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
index 8f76230..bba7692 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
@@ -69,7 +69,8 @@ public class AddForeignKeyHandler extends AbstractMessageHandler {
AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), fks,
new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec());
- Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(
+ new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
tasks.add(addConstraintsTask);
context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
index fa09f59..90d9008 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
@@ -72,7 +72,8 @@ public class AddNotNullConstraintHandler extends AbstractMessageHandler {
nns, new ArrayList<SQLDefaultConstraint>(),
new ArrayList<SQLCheckConstraint>(),
context.eventOnlyReplicationSpec());
- Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(
+ new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
tasks.add(addConstraintsTask);
context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
index f9a615a..e8966ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
@@ -64,7 +64,8 @@ public class AddPrimaryKeyHandler extends AbstractMessageHandler {
AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, pks, new ArrayList<SQLForeignKey>(),
new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec());
- Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(
+ new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
tasks.add(addConstraintsTask);
context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
index 757381a..81f1c5a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
@@ -64,7 +64,8 @@ public class AddUniqueConstraintHandler extends AbstractMessageHandler {
AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), new ArrayList<SQLForeignKey>(),
uks, context.eventOnlyReplicationSpec());
- Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+ Task<DDLWork> addConstraintsTask = TaskFactory.get(
+ new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
tasks.add(addConstraintsTask);
context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
index 6c6ee02..00ce977 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
@@ -80,7 +80,7 @@ public class AlterDatabaseHandler extends AbstractMessageHandler {
}
Task<DDLWork> alterDbTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, alterDbDesc));
+ new DDLWork(readEntitySet, writeEntitySet, alterDbDesc), context.hiveConf);
context.log.debug("Added alter database task : {}:{}",
alterDbTask.getId(), actualDbName);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
index 0dc72e0..f8d8d1a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
@@ -61,12 +61,12 @@ public class CreateDatabaseHandler extends AbstractMessageHandler {
new CreateDatabaseDesc(destinationDBName, db.getDescription(), null, true);
createDatabaseDesc.setDatabaseProperties(db.getParameters());
Task<DDLWork> createDBTask = TaskFactory.get(
- new DDLWork(new HashSet<>(), new HashSet<>(), createDatabaseDesc));
+ new DDLWork(new HashSet<>(), new HashSet<>(), createDatabaseDesc), context.hiveConf);
if (!db.getParameters().isEmpty()) {
AlterDatabaseDesc alterDbDesc = new AlterDatabaseDesc(destinationDBName, db.getParameters(),
context.eventOnlyReplicationSpec());
Task<DDLWork> alterDbProperties = TaskFactory
- .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc));
+ .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc), context.hiveConf);
createDBTask.addDependentTask(alterDbProperties);
}
if (StringUtils.isNotEmpty(db.getOwnerName())) {
@@ -74,7 +74,7 @@ public class CreateDatabaseHandler extends AbstractMessageHandler {
new PrincipalDesc(db.getOwnerName(), db.getOwnerType()),
context.eventOnlyReplicationSpec());
Task<DDLWork> alterDbTask = TaskFactory
- .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbOwner));
+ .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbOwner), context.hiveConf);
createDBTask.addDependentTask(alterDbTask);
}
updatedMetadata
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
index 77c2dd2..f7c9040 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
@@ -65,7 +65,7 @@ public class CreateFunctionHandler extends AbstractMessageHandler {
context.log.debug("Loading function desc : {}", descToLoad.toString());
Task<FunctionWork> createTask = TaskFactory.get(
- new FunctionWork(descToLoad));
+ new FunctionWork(descToLoad), context.hiveConf);
context.log.debug("Added create function task : {}:{},{}", createTask.getId(),
descToLoad.getFunctionName(), descToLoad.getClassName());
// This null check is specifically done as the same class is used to handle both incremental and
@@ -92,7 +92,7 @@ public class CreateFunctionHandler extends AbstractMessageHandler {
* which should only happen when the last task is finished, at which point the child of the barrier task is picked up.
*/
Task<? extends Serializable> barrierTask =
- TaskFactory.get(new DependencyCollectionWork());
+ TaskFactory.get(new DependencyCollectionWork(), context.hiveConf);
builder.replCopyTasks.forEach(t -> t.addDependentTask(barrierTask));
barrierTask.addDependentTask(createTask);
return builder.replCopyTasks;
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
index 7c7ce1d..5f9f879 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
@@ -39,7 +39,8 @@ public class DropConstraintHandler extends AbstractMessageHandler {
AlterTableDesc dropConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, constraintName,
context.eventOnlyReplicationSpec());
- Task<DDLWork> dropConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, dropConstraintsDesc));
+ Task<DDLWork> dropConstraintsTask = TaskFactory.get(
+ new DDLWork(readEntitySet, writeEntitySet, dropConstraintsDesc), context.hiveConf);
context.log.debug("Added drop constrain task : {}:{}", dropConstraintsTask.getId(), actualTblName);
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);
return Collections.singletonList(dropConstraintsTask);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
index 363f08c..8b11a9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
@@ -40,7 +40,7 @@ public class DropDatabaseHandler extends AbstractMessageHandler {
new DropDatabaseDesc(actualDbName, true, context.eventOnlyReplicationSpec());
Task<? extends Serializable> dropDBTask =
TaskFactory
- .get(new DDLWork(new HashSet<>(), new HashSet<>(), desc));
+ .get(new DDLWork(new HashSet<>(), new HashSet<>(), desc), context.hiveConf);
context.log.info(
"Added drop database task : {}:{}", dropDBTask.getId(), desc.getDatabaseName());
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, null, null);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
index 1fc7e13..fee2bb5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
@@ -39,7 +39,7 @@ public class DropFunctionHandler extends AbstractMessageHandler {
FunctionUtils.qualifyFunctionName(msg.getFunctionName(), actualDbName);
DropFunctionDesc desc = new DropFunctionDesc(
qualifiedFunctionName, false, context.eventOnlyReplicationSpec());
- Task<FunctionWork> dropFunctionTask = TaskFactory.get(new FunctionWork(desc));
+ Task<FunctionWork> dropFunctionTask = TaskFactory.get(new FunctionWork(desc), context.hiveConf);
context.log.debug(
"Added drop function task : {}:{}", dropFunctionTask.getId(), desc.getFunctionName()
);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
index 1a28eec..7281a1c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
@@ -53,7 +53,7 @@ public class DropPartitionHandler extends AbstractMessageHandler {
DropTableDesc dropPtnDesc = new DropTableDesc(actualDbName + "." + actualTblName,
partSpecs, null, true, context.eventOnlyReplicationSpec());
Task<DDLWork> dropPtnTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, dropPtnDesc)
+ new DDLWork(readEntitySet, writeEntitySet, dropPtnDesc), context.hiveConf
);
context.log.debug("Added drop ptn task : {}:{},{}", dropPtnTask.getId(),
dropPtnDesc.getTableName(), msg.getPartitions());
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
index 3e567e9..62784e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
@@ -40,7 +40,7 @@ public class DropTableHandler extends AbstractMessageHandler {
null, true, true, context.eventOnlyReplicationSpec(), false
);
Task<DDLWork> dropTableTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, dropTableDesc)
+ new DDLWork(readEntitySet, writeEntitySet, dropTableDesc), context.hiveConf
);
context.log.debug(
"Added drop tbl task : {}:{}", dropTableTask.getId(), dropTableDesc.getTableName()
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
index 396b7ee..43f2cbc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
@@ -62,7 +62,7 @@ public class RenamePartitionHandler extends AbstractMessageHandler {
RenamePartitionDesc renamePtnDesc = new RenamePartitionDesc(
tableName, oldPartSpec, newPartSpec, context.eventOnlyReplicationSpec());
Task<DDLWork> renamePtnTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, renamePtnDesc));
+ new DDLWork(readEntitySet, writeEntitySet, renamePtnDesc), context.hiveConf);
context.log.debug("Added rename ptn task : {}:{}->{}",
renamePtnTask.getId(), oldPartSpec, newPartSpec);
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, newPartSpec);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
index 98bf625..83433d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
@@ -61,7 +61,7 @@ public class RenameTableHandler extends AbstractMessageHandler {
AlterTableDesc renameTableDesc = new AlterTableDesc(
oldName, newName, false, context.eventOnlyReplicationSpec());
Task<DDLWork> renameTableTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, renameTableDesc));
+ new DDLWork(readEntitySet, writeEntitySet, renameTableDesc), context.hiveConf);
context.log.debug("Added rename table task : {}:{}->{}",
renameTableTask.getId(), oldName, newName);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
index 1e7fa10..b4fc000 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
@@ -58,7 +58,7 @@ public class TruncatePartitionHandler extends AbstractMessageHandler {
actualDbName + "." + actualTblName, partSpec,
context.eventOnlyReplicationSpec());
Task<DDLWork> truncatePtnTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc));
+ new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc), context.hiveConf);
context.log.debug("Added truncate ptn task : {}:{}", truncatePtnTask.getId(),
truncateTableDesc.getTableName());
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, partSpec);
http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
index bdef67f..fe73a18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
@@ -39,7 +39,7 @@ public class TruncateTableHandler extends AbstractMessageHandler {
actualDbName + "." + actualTblName,
null, context.eventOnlyReplicationSpec());
Task<DDLWork> truncateTableTask = TaskFactory.get(
- new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc));
+ new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc), context.hiveConf);
context.log.debug("Added truncate tbl task : {}:{}", truncateTableTask.getId(),
truncateTableDesc.getTableName());