You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2018/03/28 16:51:14 UTC

hive git commit: HIVE-19007: Support REPL LOAD from primary using replica connection configurations received through WITH clause (Sankar Hariappan, reviewed by Thejas Nair)

Repository: hive
Updated Branches:
  refs/heads/master 9b84ed4b2 -> 6edff8a91


HIVE-19007: Support REPL LOAD from primary using replica connection configurations received through WITH clause (Sankar Hariappan, reviewed by Thejas Nair)


Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6edff8a9
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6edff8a9
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6edff8a9

Branch: refs/heads/master
Commit: 6edff8a91fa4db1ba0450f81fdf1c1c1beec50d5
Parents: 9b84ed4
Author: Sankar Hariappan <sa...@apache.org>
Authored: Wed Mar 28 22:20:29 2018 +0530
Committer: Sankar Hariappan <sa...@apache.org>
Committed: Wed Mar 28 22:20:29 2018 +0530

----------------------------------------------------------------------
 ...TestReplicationScenariosAcrossInstances.java | 96 ++++++++++++++++++++
 .../hadoop/hive/ql/parse/WarehouseInstance.java | 23 +++++
 .../hive/ql/parse/ImportSemanticAnalyzer.java   | 36 ++++----
 .../ql/parse/ReplicationSemanticAnalyzer.java   | 36 +++++---
 .../repl/load/message/AddForeignKeyHandler.java |  3 +-
 .../message/AddNotNullConstraintHandler.java    |  3 +-
 .../repl/load/message/AddPrimaryKeyHandler.java |  3 +-
 .../message/AddUniqueConstraintHandler.java     |  3 +-
 .../repl/load/message/AlterDatabaseHandler.java |  2 +-
 .../load/message/CreateDatabaseHandler.java     |  6 +-
 .../load/message/CreateFunctionHandler.java     |  4 +-
 .../load/message/DropConstraintHandler.java     |  3 +-
 .../repl/load/message/DropDatabaseHandler.java  |  2 +-
 .../repl/load/message/DropFunctionHandler.java  |  2 +-
 .../repl/load/message/DropPartitionHandler.java |  2 +-
 .../repl/load/message/DropTableHandler.java     |  2 +-
 .../load/message/RenamePartitionHandler.java    |  2 +-
 .../repl/load/message/RenameTableHandler.java   |  2 +-
 .../load/message/TruncatePartitionHandler.java  |  2 +-
 .../repl/load/message/TruncateTableHandler.java |  2 +-
 20 files changed, 181 insertions(+), 53 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
index 13b918d..689ca76 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationScenariosAcrossInstances.java
@@ -524,4 +524,100 @@ public class TestReplicationScenariosAcrossInstances {
     */
 
   }
+
+  @Test
+  public void testReplLoadFromSourceUsingWithClause() throws Throwable {
+    HiveConf replicaConf = replica.getConf();
+    List<String> withConfigs = Arrays.asList(
+            "'hive.metastore.warehouse.dir'='" + replicaConf.getVar(HiveConf.ConfVars.METASTOREWAREHOUSE) + "'",
+            "'hive.metastore.uris'='" + replicaConf.getVar(HiveConf.ConfVars.METASTOREURIS) + "'",
+            "'hive.repl.replica.functions.root.dir'='" + replicaConf.getVar(HiveConf.ConfVars.REPL_FUNCTIONS_ROOT_DIR) + "'");
+
+    ////////////  Bootstrap   ////////////
+    WarehouseInstance.Tuple bootstrapTuple = primary
+            .run("use " + primaryDbName)
+            .run("create table table1 (i int)")
+            .run("create table table2 (id int) partitioned by (country string)")
+            .run("insert into table1 values (1)")
+            .dump(primaryDbName, null);
+
+    // Run load on primary itself
+    primary.load(replicatedDbName, bootstrapTuple.dumpLocation, withConfigs)
+            .status(replicatedDbName, withConfigs)
+            .verifyResult(bootstrapTuple.lastReplicationId);
+
+    replica.run("use " + replicatedDbName)
+            .run("show tables")
+            .verifyResults(new String[] { "table1", "table2" })
+            .run("select * from table1")
+            .verifyResults(new String[]{ "1" });
+
+    ////////////  First Incremental ////////////
+    WarehouseInstance.Tuple incrementalOneTuple = primary
+                    .run("use " + primaryDbName)
+                    .run("alter table table1 rename to renamed_table1")
+                    .run("insert into table2 partition(country='india') values (1) ")
+                    .run("insert into table2 partition(country='usa') values (2) ")
+                    .run("create table table3 (i int)")
+                    .run("insert into table3 values(10)")
+                    .run("create function " + primaryDbName
+                      + ".testFunctionOne as 'hivemall.tools.string.StopwordUDF' "
+                      + "using jar  'ivy://io.github.myui:hivemall:0.4.0-2'")
+                    .dump(primaryDbName, bootstrapTuple.lastReplicationId);
+
+    // Run load on primary itself
+    primary.load(replicatedDbName, incrementalOneTuple.dumpLocation, withConfigs)
+            .status(replicatedDbName, withConfigs)
+            .verifyResult(incrementalOneTuple.lastReplicationId);
+
+    replica.run("use " + replicatedDbName)
+            .run("show tables")
+            .verifyResults(new String[] { "renamed_table1", "table2", "table3" })
+            .run("select * from renamed_table1")
+            .verifyResults(new String[] { "1" })
+            .run("select id from table2 order by id")
+            .verifyResults(new String[] { "1", "2" })
+            .run("select * from table3")
+            .verifyResults(new String[] { "10" })
+            .run("show functions like '" + replicatedDbName + "*'")
+            .verifyResult(replicatedDbName + ".testFunctionOne");
+
+    ////////////  Second Incremental ////////////
+    WarehouseInstance.Tuple secondIncremental = primary
+            .run("use " + primaryDbName)
+            .run("alter table table2 add columns (zipcode int)")
+            .run("alter table table3 set tblproperties('custom.property'='custom.value')")
+            .run("drop table renamed_table1")
+            .run("alter table table2 drop partition(country='usa')")
+            .run("truncate table table3")
+            .run("drop function " + primaryDbName + ".testFunctionOne ")
+            .dump(primaryDbName, incrementalOneTuple.lastReplicationId);
+
+    // Run load on primary itself
+    primary.load(replicatedDbName, secondIncremental.dumpLocation, withConfigs)
+            .status(replicatedDbName, withConfigs)
+            .verifyResult(secondIncremental.lastReplicationId);
+
+    replica.run("use " + replicatedDbName)
+            .run("show tables")
+            .verifyResults(new String[] { "table2", "table3"})
+            .run("desc table2")
+            .verifyResults(new String[] {
+                    "id                  \tint                 \t                    ",
+                    "country             \tstring              \t                    ",
+                    "zipcode             \tint                 \t                    ",
+                    "\t \t ",
+                    "# Partition Information\t \t ",
+                    "# col_name            \tdata_type           \tcomment             ",
+                    "country             \tstring              \t                    ",
+            })
+            .run("show tblproperties table3('custom.property')")
+            .verifyResults(new String[] { "custom.value\t " })
+            .run("select id from table2 order by id")
+            .verifyResults(new String[] { "1" })
+            .run("select * from table3")
+            .verifyResults(Collections.emptyList())
+            .run("show functions like '" + replicatedDbName + "*'")
+            .verifyResult(null);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
index bc70fe4..0006ea5 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/WarehouseInstance.java
@@ -146,6 +146,10 @@ public class WarehouseInstance implements Closeable {
     return PathBuilder.fullyQualifiedHDFSUri(path, fs);
   }
 
+  public HiveConf getConf() {
+    return hiveConf;
+  }
+
   private int next = 0;
 
   private void advanceDumpDir() {
@@ -201,6 +205,25 @@ public class WarehouseInstance implements Closeable {
     return this;
   }
 
+  WarehouseInstance load(String replicatedDbName, String dumpLocation, List<String> withClauseOptions)
+          throws Throwable {
+    String replLoadCmd = "REPL LOAD " + replicatedDbName + " FROM '" + dumpLocation + "'";
+    if (!withClauseOptions.isEmpty()) {
+      replLoadCmd += " WITH (" + StringUtils.join(withClauseOptions, ",") + ")";
+    }
+    run("EXPLAIN " + replLoadCmd);
+    printOutput();
+    return run(replLoadCmd);
+  }
+
+  WarehouseInstance status(String replicatedDbName, List<String> withClauseOptions) throws Throwable {
+    String replStatusCmd = "REPL STATUS " + replicatedDbName;
+    if (!withClauseOptions.isEmpty()) {
+      replStatusCmd += " WITH (" + StringUtils.join(withClauseOptions, ",") + ")";
+    }
+    return run(replStatusCmd);
+  }
+
   WarehouseInstance verifyResult(String data) throws IOException {
     verifyResults(data == null ? new String[] {} : new String[] { data });
     return this;

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 9a91e3f..e862589 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -424,7 +424,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
         loadPath, Utilities.getTableDesc(table), new TreeMap<>(), lft, writeId);
     loadTableWork.setStmtId(stmtId);
     MoveWork mv = new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false);
-    Task<?> loadTableTask = TaskFactory.get(mv);
+    Task<?> loadTableTask = TaskFactory.get(mv, x.getConf());
     copyTask.addDependentTask(loadTableTask);
     x.getTasks().add(copyTask);
     return loadTableTask;
@@ -466,11 +466,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
     } else {
       partSpec.setLocation(ptn.getLocation()); // use existing location
     }
-    return TaskFactory.get(new DDLWork(
-        x.getInputs(),
-        x.getOutputs(),
-        addPartitionDesc
-    ));
+    return TaskFactory.get(new DDLWork(x.getInputs(), x.getOutputs(), addPartitionDesc), x.getConf());
   }
 
  private static Task<?> addSinglePartition(URI fromURI, FileSystem fs, ImportTableDesc tblDesc,
@@ -484,8 +480,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
           + partSpecToString(partSpec.getPartSpec()));
       // addPartitionDesc already has the right partition location
       @SuppressWarnings("unchecked")
-      Task<?> addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
-          x.getOutputs(), addPartitionDesc));
+      Task<?> addPartTask = TaskFactory.get(
+              new DDLWork(x.getInputs(), x.getOutputs(), addPartitionDesc), x.getConf());
       return addPartTask;
     } else {
       String srcLocation = partSpec.getLocation();
@@ -518,8 +514,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
         copyTask = TaskFactory.get(cw);
       }
 
-      Task<?> addPartTask = TaskFactory.get(new DDLWork(x.getInputs(),
-          x.getOutputs(), addPartitionDesc));
+      Task<?> addPartTask = TaskFactory.get(
+              new DDLWork(x.getInputs(), x.getOutputs(), addPartitionDesc), x.getConf());
+
       // Note: this sets LoadFileType incorrectly for ACID; is that relevant for import?
       //       See setLoadFileType and setIsAcidIow calls elsewhere for an example.
       LoadTableDesc loadTableWork = new LoadTableDesc(moveTaskSrc, Utilities.getTableDesc(table),
@@ -528,8 +525,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
               writeId);
       loadTableWork.setStmtId(stmtId);
       loadTableWork.setInheritTableSpecs(false);
-      Task<?> loadPartTask = TaskFactory.get(new MoveWork(
-          x.getInputs(), x.getOutputs(), loadTableWork, null, false));
+      Task<?> loadPartTask = TaskFactory.get(
+              new MoveWork(x.getInputs(), x.getOutputs(), loadTableWork, null, false),
+              x.getConf());
       copyTask.addDependentTask(loadPartTask);
       addPartTask.addDependentTask(loadPartTask);
       x.getTasks().add(copyTask);
@@ -830,7 +828,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
     if (table != null) {
       if (table.isPartitioned()) {
         x.getLOG().debug("table partitioned");
-        Task<?> ict = createImportCommitTask(
+        Task<?> ict = createImportCommitTask(x.getConf(),
             table.getDbName(), table.getTableName(), writeId, stmtId,
             AcidUtils.isInsertOnlyTable(table.getParameters()));
 
@@ -867,7 +865,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
       x.getOutputs().add(new WriteEntity(parentDb, WriteEntity.WriteType.DDL_SHARED));
 
       if (isPartitioned(tblDesc)) {
-        Task<?> ict = createImportCommitTask(
+        Task<?> ict = createImportCommitTask(x.getConf(),
             tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
             AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
         for (AddPartitionDesc addPartitionDesc : partitionDescs) {
@@ -903,10 +901,10 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
   }
 
   private static Task<?> createImportCommitTask(
-      String dbName, String tblName, Long writeId, int stmtId, boolean isMmTable) {
+      HiveConf conf, String dbName, String tblName, Long writeId, int stmtId, boolean isMmTable) {
     // TODO: noop, remove?
     Task<ImportCommitWork> ict = (!isMmTable) ? null : TaskFactory.get(
-        new ImportCommitWork(dbName, tblName, writeId, stmtId));
+        new ImportCommitWork(dbName, tblName, writeId, stmtId), conf);
     return ict;
   }
 
@@ -994,7 +992,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
 
       if (!replicationSpec.isMetadataOnly()) {
         if (isPartitioned(tblDesc)) {
-          Task<?> ict = createImportCommitTask(
+          Task<?> ict = createImportCommitTask(x.getConf(),
               tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
               AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
           for (AddPartitionDesc addPartitionDesc : partitionDescs) {
@@ -1021,8 +1019,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
           Map<String, String> partSpec = addPartitionDesc.getPartition(0).getPartSpec();
           org.apache.hadoop.hive.ql.metadata.Partition ptn = null;
           Task<?> ict = replicationSpec.isMetadataOnly() ? null : createImportCommitTask(
-              tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
-              AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
+                  x.getConf(), tblDesc.getDatabaseName(), tblDesc.getTableName(), writeId, stmtId,
+                  AcidUtils.isInsertOnlyTable(tblDesc.getTblProps()));
           if ((ptn = x.getHive().getPartition(table, partSpec, false)) == null) {
             if (!replicationSpec.isMetadataOnly()){
               x.getTasks().add(addSinglePartition(

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index 753f039..9f18efd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -82,8 +82,9 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
   // of any other queries running in the session
   private HiveConf conf;
 
-  // This will be set to true only if repl-status is fired with "WITH" keyword
-  private boolean needNewdb;
+  // By default, this will be same as that of super class BaseSemanticAnalyzer. But need to obtain again
+  // if the Hive configs are received from WITH clause in REPL LOAD or REPL STATUS commands.
+  private Hive db;
 
   private static String testInjectDumpDir = null; // unit tests can overwrite this to affect default dump behaviour
   private static final String dumpSchema = "dump_dir,last_repl_id#string,string";
@@ -93,6 +94,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
 
   ReplicationSemanticAnalyzer(QueryState queryState) throws SemanticException {
     super(queryState);
+    this.db = super.db;
     this.conf = new HiveConf(super.conf);
   }
 
@@ -226,6 +228,13 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
             for (Map.Entry<String, String> config : replConfigs.entrySet()) {
               conf.set(config.getKey(), config.getValue());
             }
+
+            // As hive conf is changed, need to get the Hive DB again with it.
+            try {
+              db = Hive.get(conf);
+            } catch (HiveException e) {
+              throw new SemanticException(e);
+            }
           }
           break;
         default:
@@ -420,7 +429,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
           Map<String, String> dbProps = new HashMap<>();
           dbProps.put(ReplicationSpec.KEY.CURR_STATE_ID.toString(), String.valueOf(dmd.getEventTo()));
           ReplStateLogWork replStateLogWork = new ReplStateLogWork(replLogger, dbProps);
-          Task<? extends Serializable> barrierTask = TaskFactory.get(replStateLogWork);
+          Task<? extends Serializable> barrierTask = TaskFactory.get(replStateLogWork, conf);
           taskChainTail.addDependentTask(barrierTask);
           LOG.debug("Added {}:{} as a precursor of barrier task {}:{}",
                   taskChainTail.getClass(), taskChainTail.getId(),
@@ -560,7 +569,6 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
 
   // REPL STATUS
   private void initReplStatus(ASTNode ast) throws SemanticException{
-    needNewdb = false;
     dbNameOrPattern = PlanUtils.stripQuotes(ast.getChild(0).getText());
     int numChildren = ast.getChildCount();
     for (int i = 1; i < numChildren; i++) {
@@ -576,8 +584,14 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
           for (Map.Entry<String, String> config : replConfigs.entrySet()) {
             conf.set(config.getKey(), config.getValue());
           }
+
+          // As hive conf is changed, need to get the Hive DB again with it.
+          try {
+            db = Hive.get(conf);
+          } catch (HiveException e) {
+            throw new SemanticException(e);
+          }
         }
-        needNewdb = true;
         break;
       default:
         throw new SemanticException("Unrecognized token in REPL STATUS statement");
@@ -592,16 +606,9 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
     String replLastId = null;
 
     try {
-      Hive newDb;
-      if (needNewdb) {
-        newDb = Hive.get(conf, false);
-      } else {
-        newDb = db;
-      }
-
       if (tblNameOrPattern != null) {
         // Checking for status of table
-        Table tbl = newDb.getTable(dbNameOrPattern, tblNameOrPattern);
+        Table tbl = db.getTable(dbNameOrPattern, tblNameOrPattern);
         if (tbl != null) {
           inputs.add(new ReadEntity(tbl));
           Map<String, String> params = tbl.getParameters();
@@ -611,8 +618,7 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
         }
       } else {
         // Checking for status of a db
-
-        Database database = newDb.getDatabase(dbNameOrPattern);
+        Database database = db.getDatabase(dbNameOrPattern);
         if (database != null) {
           inputs.add(new ReadEntity(database));
           Map<String, String> params = database.getParameters();

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
index 8f76230..bba7692 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddForeignKeyHandler.java
@@ -69,7 +69,8 @@ public class AddForeignKeyHandler extends AbstractMessageHandler {
 
     AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), fks,
         new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec());
-    Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+    Task<DDLWork> addConstraintsTask = TaskFactory.get(
+            new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
     tasks.add(addConstraintsTask);
     context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
index fa09f59..90d9008 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddNotNullConstraintHandler.java
@@ -72,7 +72,8 @@ public class AddNotNullConstraintHandler extends AbstractMessageHandler {
                                                            nns, new ArrayList<SQLDefaultConstraint>(),
                                                            new ArrayList<SQLCheckConstraint>(),
                                                            context.eventOnlyReplicationSpec());
-    Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+    Task<DDLWork> addConstraintsTask = TaskFactory.get(
+            new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
     tasks.add(addConstraintsTask);
     context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
index f9a615a..e8966ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddPrimaryKeyHandler.java
@@ -64,7 +64,8 @@ public class AddPrimaryKeyHandler extends AbstractMessageHandler {
 
     AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, pks, new ArrayList<SQLForeignKey>(),
         new ArrayList<SQLUniqueConstraint>(), context.eventOnlyReplicationSpec());
-    Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+    Task<DDLWork> addConstraintsTask = TaskFactory.get(
+            new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
     tasks.add(addConstraintsTask);
     context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
index 757381a..81f1c5a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AddUniqueConstraintHandler.java
@@ -64,7 +64,8 @@ public class AddUniqueConstraintHandler extends AbstractMessageHandler {
 
     AlterTableDesc addConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, new ArrayList<SQLPrimaryKey>(), new ArrayList<SQLForeignKey>(),
         uks, context.eventOnlyReplicationSpec());
-    Task<DDLWork> addConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc));
+    Task<DDLWork> addConstraintsTask = TaskFactory.get(
+            new DDLWork(readEntitySet, writeEntitySet, addConstraintsDesc), context.hiveConf);
     tasks.add(addConstraintsTask);
     context.log.debug("Added add constrains task : {}:{}", addConstraintsTask.getId(), actualTblName);
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
index 6c6ee02..00ce977 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/AlterDatabaseHandler.java
@@ -80,7 +80,7 @@ public class AlterDatabaseHandler extends AbstractMessageHandler {
       }
 
       Task<DDLWork> alterDbTask = TaskFactory.get(
-          new DDLWork(readEntitySet, writeEntitySet, alterDbDesc));
+          new DDLWork(readEntitySet, writeEntitySet, alterDbDesc), context.hiveConf);
       context.log.debug("Added alter database task : {}:{}",
               alterDbTask.getId(), actualDbName);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
index 0dc72e0..f8d8d1a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateDatabaseHandler.java
@@ -61,12 +61,12 @@ public class CreateDatabaseHandler extends AbstractMessageHandler {
         new CreateDatabaseDesc(destinationDBName, db.getDescription(), null, true);
     createDatabaseDesc.setDatabaseProperties(db.getParameters());
     Task<DDLWork> createDBTask = TaskFactory.get(
-        new DDLWork(new HashSet<>(), new HashSet<>(), createDatabaseDesc));
+        new DDLWork(new HashSet<>(), new HashSet<>(), createDatabaseDesc), context.hiveConf);
     if (!db.getParameters().isEmpty()) {
       AlterDatabaseDesc alterDbDesc = new AlterDatabaseDesc(destinationDBName, db.getParameters(),
           context.eventOnlyReplicationSpec());
       Task<DDLWork> alterDbProperties = TaskFactory
-          .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc));
+          .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbDesc), context.hiveConf);
       createDBTask.addDependentTask(alterDbProperties);
     }
     if (StringUtils.isNotEmpty(db.getOwnerName())) {
@@ -74,7 +74,7 @@ public class CreateDatabaseHandler extends AbstractMessageHandler {
           new PrincipalDesc(db.getOwnerName(), db.getOwnerType()),
           context.eventOnlyReplicationSpec());
       Task<DDLWork> alterDbTask = TaskFactory
-          .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbOwner));
+          .get(new DDLWork(new HashSet<>(), new HashSet<>(), alterDbOwner), context.hiveConf);
       createDBTask.addDependentTask(alterDbTask);
     }
     updatedMetadata

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
index 77c2dd2..f7c9040 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/CreateFunctionHandler.java
@@ -65,7 +65,7 @@ public class CreateFunctionHandler extends AbstractMessageHandler {
 
       context.log.debug("Loading function desc : {}", descToLoad.toString());
       Task<FunctionWork> createTask = TaskFactory.get(
-          new FunctionWork(descToLoad));
+          new FunctionWork(descToLoad), context.hiveConf);
       context.log.debug("Added create function task : {}:{},{}", createTask.getId(),
           descToLoad.getFunctionName(), descToLoad.getClassName());
       // This null check is specifically done as the same class is used to handle both incremental and
@@ -92,7 +92,7 @@ public class CreateFunctionHandler extends AbstractMessageHandler {
          *  which should only happen when the last task is finished, at which point the child of the barrier task is picked up.
          */
         Task<? extends Serializable> barrierTask =
-            TaskFactory.get(new DependencyCollectionWork());
+            TaskFactory.get(new DependencyCollectionWork(), context.hiveConf);
         builder.replCopyTasks.forEach(t -> t.addDependentTask(barrierTask));
         barrierTask.addDependentTask(createTask);
         return builder.replCopyTasks;

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
index 7c7ce1d..5f9f879 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropConstraintHandler.java
@@ -39,7 +39,8 @@ public class DropConstraintHandler extends AbstractMessageHandler {
 
     AlterTableDesc dropConstraintsDesc = new AlterTableDesc(actualDbName + "." + actualTblName, constraintName,
         context.eventOnlyReplicationSpec());
-    Task<DDLWork> dropConstraintsTask = TaskFactory.get(new DDLWork(readEntitySet, writeEntitySet, dropConstraintsDesc));
+    Task<DDLWork> dropConstraintsTask = TaskFactory.get(
+            new DDLWork(readEntitySet, writeEntitySet, dropConstraintsDesc), context.hiveConf);
     context.log.debug("Added drop constrain task : {}:{}", dropConstraintsTask.getId(), actualTblName);
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);
     return Collections.singletonList(dropConstraintsTask);    

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
index 363f08c..8b11a9e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropDatabaseHandler.java
@@ -40,7 +40,7 @@ public class DropDatabaseHandler extends AbstractMessageHandler {
         new DropDatabaseDesc(actualDbName, true, context.eventOnlyReplicationSpec());
     Task<? extends Serializable> dropDBTask =
         TaskFactory
-            .get(new DDLWork(new HashSet<>(), new HashSet<>(), desc));
+            .get(new DDLWork(new HashSet<>(), new HashSet<>(), desc), context.hiveConf);
     context.log.info(
         "Added drop database task : {}:{}", dropDBTask.getId(), desc.getDatabaseName());
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, null, null);

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
index 1fc7e13..fee2bb5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropFunctionHandler.java
@@ -39,7 +39,7 @@ public class DropFunctionHandler extends AbstractMessageHandler {
         FunctionUtils.qualifyFunctionName(msg.getFunctionName(), actualDbName);
     DropFunctionDesc desc = new DropFunctionDesc(
             qualifiedFunctionName, false, context.eventOnlyReplicationSpec());
-    Task<FunctionWork> dropFunctionTask = TaskFactory.get(new FunctionWork(desc));
+    Task<FunctionWork> dropFunctionTask = TaskFactory.get(new FunctionWork(desc), context.hiveConf);
     context.log.debug(
         "Added drop function task : {}:{}", dropFunctionTask.getId(), desc.getFunctionName()
     );

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
index 1a28eec..7281a1c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
@@ -53,7 +53,7 @@ public class DropPartitionHandler extends AbstractMessageHandler {
         DropTableDesc dropPtnDesc = new DropTableDesc(actualDbName + "." + actualTblName,
             partSpecs, null, true, context.eventOnlyReplicationSpec());
         Task<DDLWork> dropPtnTask = TaskFactory.get(
-            new DDLWork(readEntitySet, writeEntitySet, dropPtnDesc)
+            new DDLWork(readEntitySet, writeEntitySet, dropPtnDesc), context.hiveConf
         );
         context.log.debug("Added drop ptn task : {}:{},{}", dropPtnTask.getId(),
             dropPtnDesc.getTableName(), msg.getPartitions());

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
index 3e567e9..62784e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropTableHandler.java
@@ -40,7 +40,7 @@ public class DropTableHandler extends AbstractMessageHandler {
         null, true, true, context.eventOnlyReplicationSpec(), false
     );
     Task<DDLWork> dropTableTask = TaskFactory.get(
-        new DDLWork(readEntitySet, writeEntitySet, dropTableDesc)
+        new DDLWork(readEntitySet, writeEntitySet, dropTableDesc), context.hiveConf
     );
     context.log.debug(
         "Added drop tbl task : {}:{}", dropTableTask.getId(), dropTableDesc.getTableName()

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
index 396b7ee..43f2cbc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
@@ -62,7 +62,7 @@ public class RenamePartitionHandler extends AbstractMessageHandler {
     RenamePartitionDesc renamePtnDesc = new RenamePartitionDesc(
             tableName, oldPartSpec, newPartSpec, context.eventOnlyReplicationSpec());
     Task<DDLWork> renamePtnTask = TaskFactory.get(
-        new DDLWork(readEntitySet, writeEntitySet, renamePtnDesc));
+        new DDLWork(readEntitySet, writeEntitySet, renamePtnDesc), context.hiveConf);
     context.log.debug("Added rename ptn task : {}:{}->{}",
                       renamePtnTask.getId(), oldPartSpec, newPartSpec);
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, newPartSpec);

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
index 98bf625..83433d7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
@@ -61,7 +61,7 @@ public class RenameTableHandler extends AbstractMessageHandler {
       AlterTableDesc renameTableDesc = new AlterTableDesc(
               oldName, newName, false, context.eventOnlyReplicationSpec());
       Task<DDLWork> renameTableTask = TaskFactory.get(
-          new DDLWork(readEntitySet, writeEntitySet, renameTableDesc));
+          new DDLWork(readEntitySet, writeEntitySet, renameTableDesc), context.hiveConf);
       context.log.debug("Added rename table task : {}:{}->{}",
                         renameTableTask.getId(), oldName, newName);
 

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
index 1e7fa10..b4fc000 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
@@ -58,7 +58,7 @@ public class TruncatePartitionHandler extends AbstractMessageHandler {
             actualDbName + "." + actualTblName, partSpec,
             context.eventOnlyReplicationSpec());
     Task<DDLWork> truncatePtnTask = TaskFactory.get(
-        new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc));
+        new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc), context.hiveConf);
     context.log.debug("Added truncate ptn task : {}:{}", truncatePtnTask.getId(),
         truncateTableDesc.getTableName());
     updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, partSpec);

http://git-wip-us.apache.org/repos/asf/hive/blob/6edff8a9/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
index bdef67f..fe73a18 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
@@ -39,7 +39,7 @@ public class TruncateTableHandler extends AbstractMessageHandler {
             actualDbName + "." + actualTblName,
             null, context.eventOnlyReplicationSpec());
     Task<DDLWork> truncateTableTask = TaskFactory.get(
-        new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc));
+        new DDLWork(readEntitySet, writeEntitySet, truncateTableDesc), context.hiveConf);
 
     context.log.debug("Added truncate tbl task : {}:{}", truncateTableTask.getId(),
         truncateTableDesc.getTableName());