You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ma...@apache.org on 2018/12/10 12:11:16 UTC
[09/11] hive git commit: HIVE-20966 : Support bootstrap and
incremental replication to a target with hive.strict.managed.tables
enabled.(Mahesh Kumar Behera reviewed by Sankar Hariappan)
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
index b1f731f..7d2b616 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/util/ReplUtils.java
@@ -18,36 +18,51 @@
package org.apache.hadoop.hive.ql.exec.repl.util;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.TableType;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.repl.ReplStateLogWork;
+import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.DDLSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.repl.ReplLogger;
import org.apache.hadoop.hive.ql.plan.AlterTableDesc;
import org.apache.hadoop.hive.ql.plan.DDLWork;
+import org.apache.hadoop.hive.ql.plan.ReplTxnWork;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.plan.ImportTableDesc;
import org.apache.hadoop.hive.ql.stats.StatsUtils;
+import org.apache.hadoop.hive.ql.util.HiveStrictManagedMigration;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.ql.parse.repl.load.UpdatedMetaDataTracker;
+import org.apache.thrift.TException;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.io.Serializable;
+
+import static org.apache.hadoop.hive.ql.util.HiveStrictManagedMigration.TableMigrationOption.MANAGED;
public class ReplUtils {
public static final String REPL_CHECKPOINT_KEY = "hive.repl.ckpt.key";
+ // write id allocated in the current execution context which will be passed through config to be used by different
+ // tasks.
+ public static final String REPL_CURRENT_TBL_WRITE_ID = "hive.repl.current.table.write.id";
+
+
/**
* Bootstrap REPL LOAD operation type on the examined object based on ckpt state.
*/
@@ -121,4 +136,31 @@ public class ReplUtils {
}
return false;
}
+
+ public static List<Task<? extends Serializable>> addOpenTxnTaskForMigration(String actualDbName,
+ String actualTblName, HiveConf conf,
+ UpdatedMetaDataTracker updatedMetaDataTracker,
+ Task<? extends Serializable> childTask,
+ org.apache.hadoop.hive.metastore.api.Table tableObj)
+ throws IOException, TException {
+ List<Task<? extends Serializable>> taskList = new ArrayList<>();
+ taskList.add(childTask);
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVE_STRICT_MANAGED_TABLES) && updatedMetaDataTracker != null &&
+ !AcidUtils.isTransactionalTable(tableObj) &&
+ TableType.valueOf(tableObj.getTableType()) == TableType.MANAGED_TABLE) {
+ //TODO : isPathOwnByHive is hard coded to true, need to get it from repl dump metadata.
+ HiveStrictManagedMigration.TableMigrationOption migrationOption =
+ HiveStrictManagedMigration.determineMigrationTypeAutomatically(tableObj, TableType.MANAGED_TABLE,
+ null, conf, null, true);
+ if (migrationOption == MANAGED) {
+ //if conversion to managed table.
+ Task<? extends Serializable> replTxnTask = TaskFactory.get(new ReplTxnWork(actualDbName, actualTblName,
+ ReplTxnWork.OperationType.REPL_MIGRATION_OPEN_TXN), conf);
+ replTxnTask.addDependentTask(childTask);
+ updatedMetaDataTracker.setNeedCommitTxn(true);
+ taskList.add(replTxnTask);
+ }
+ }
+ return taskList;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
index 46c51eb..9ddd30c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/lockmgr/DbTxnManager.java
@@ -442,9 +442,28 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
}
}
+ private void clearLocksAndHB() throws LockException {
+ lockMgr.clearLocalLockRecords();
+ stopHeartbeat();
+ }
+
+ private void resetTxnInfo() {
+ txnId = 0;
+ stmtId = -1;
+ numStatements = 0;
+ tableWriteIds.clear();
+ }
+
@Override
public void replCommitTxn(CommitTxnRequest rqst) throws LockException {
try {
+ if (rqst.isSetReplLastIdInfo()) {
+ if (!isTxnOpen()) {
+ throw new RuntimeException("Attempt to commit before opening a transaction");
+ }
+ // For transaction started internally by repl load command, heartbeat needs to be stopped.
+ clearLocksAndHB();
+ }
getMS().replCommitTxn(rqst);
} catch (NoSuchTxnException e) {
LOG.error("Metastore could not find " + JavaUtils.txnIdToString(rqst.getTxnid()));
@@ -456,6 +475,11 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
throw le;
} catch (TException e) {
throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(), e);
+ } finally {
+ if (rqst.isSetReplLastIdInfo()) {
+ // For transaction started internally by repl load command, needs to clear the txn info.
+ resetTxnInfo();
+ }
}
}
@@ -465,8 +489,8 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
throw new RuntimeException("Attempt to commit before opening a transaction");
}
try {
- lockMgr.clearLocalLockRecords();
- stopHeartbeat();
+ // do all new clear in clearLocksAndHB method to make sure that same code is there for replCommitTxn flow.
+ clearLocksAndHB();
LOG.debug("Committing txn " + JavaUtils.txnIdToString(txnId));
getMS().commitTxn(txnId);
} catch (NoSuchTxnException e) {
@@ -480,10 +504,8 @@ public final class DbTxnManager extends HiveTxnManagerImpl {
throw new LockException(ErrorMsg.METASTORE_COMMUNICATION_FAILED.getMsg(),
e);
} finally {
- txnId = 0;
- stmtId = -1;
- numStatements = 0;
- tableWriteIds.clear();
+ // do all new reset in resetTxnInfo method to make sure that same code is there for replCommitTxn flow.
+ resetTxnInfo();
}
}
@Override
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 768a73c..c017790 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -695,7 +695,7 @@ public class Hive {
AcidUtils.TableSnapshot tableSnapshot = null;
if (transactional) {
if (replWriteId > 0) {
- ValidWriteIdList writeIds = getMSC().getValidWriteIds(getFullTableName(dbName, tblName), replWriteId);
+ ValidWriteIdList writeIds = AcidUtils.getTableValidWriteIdListWithTxnList(conf, dbName, tblName);
tableSnapshot = new TableSnapshot(replWriteId, writeIds.writeToString());
} else {
// Make sure we pass in the names, so we can get the correct snapshot for rename table.
@@ -1903,8 +1903,17 @@ public class Hive {
inheritLocation, isSkewedStoreAsSubdir, isSrcLocal, isAcidIUDoperation,
hasFollowingStatsTask, writeId, stmtId, isInsertOverwrite, isTxnTable, newFiles);
- AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf,
- newTPart.getTable(), true);
+ AcidUtils.TableSnapshot tableSnapshot = null;
+ if (isTxnTable) {
+ if ((writeId != null) && (writeId > 0)) {
+ ValidWriteIdList writeIds = AcidUtils.getTableValidWriteIdListWithTxnList(
+ conf, tbl.getDbName(), tbl.getTableName());
+ tableSnapshot = new TableSnapshot(writeId, writeIds.writeToString());
+ } else {
+ // Make sure we pass in the names, so we can get the correct snapshot for rename table.
+ tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, tbl.getDbName(), tbl.getTableName(), true);
+ }
+ }
if (tableSnapshot != null) {
newTPart.getTPartition().setWriteId(tableSnapshot.getWriteId());
}
@@ -2854,7 +2863,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
environmentContext.putToProperties(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE);
}
- alterTable(tbl, false, environmentContext, true);
+ alterTable(tbl.getCatName(), tbl.getDbName(), tbl.getTableName(), tbl, false, environmentContext,
+ true, ((writeId == null) ? 0 : writeId));
if (AcidUtils.isTransactionalTable(tbl)) {
addWriteNotificationLog(tbl, null, newFiles, writeId);
@@ -2896,8 +2906,16 @@ private void constructOneLBLocationMap(FileStatus fSta,
int size = addPartitionDesc.getPartitionCount();
List<org.apache.hadoop.hive.metastore.api.Partition> in =
new ArrayList<org.apache.hadoop.hive.metastore.api.Partition>(size);
- AcidUtils.TableSnapshot tableSnapshot =
- AcidUtils.getTableSnapshot(conf, tbl);
+ AcidUtils.TableSnapshot tableSnapshot = AcidUtils.getTableSnapshot(conf, tbl, true);
+ long writeId;
+ String validWriteIdList;
+ if (tableSnapshot != null && tableSnapshot.getWriteId() > 0) {
+ writeId = tableSnapshot.getWriteId();
+ validWriteIdList = tableSnapshot.getValidWriteIdList();
+ } else {
+ writeId = -1;
+ validWriteIdList = null;
+ }
for (int i = 0; i < size; ++i) {
org.apache.hadoop.hive.metastore.api.Partition tmpPart =
convertAddSpecToMetaPartition(tbl, addPartitionDesc.getPartition(i), conf);
@@ -2941,7 +2959,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
out.add(new Partition(tbl, outPart));
}
getMSC().alter_partitions(addPartitionDesc.getDbName(), addPartitionDesc.getTableName(),
- partsToAlter, new EnvironmentContext(), null, -1);
+ partsToAlter, new EnvironmentContext(), validWriteIdList, writeId);
for ( org.apache.hadoop.hive.metastore.api.Partition outPart :
getMSC().getPartitionsByNames(addPartitionDesc.getDbName(), addPartitionDesc.getTableName(),part_names)){
@@ -4133,7 +4151,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
public void recycleDirToCmPath(Path dataPath, boolean isPurge) throws HiveException {
try {
CmRecycleRequest request = new CmRecycleRequest(dataPath.toString(), isPurge);
- getMSC().recycleDirToCmPath(request);
+ getSynchronizedMSC().recycleDirToCmPath(request);
} catch (Exception e) {
throw new HiveException(e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
index dd75b32..156f755 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/EximUtil.java
@@ -96,6 +96,7 @@ public class EximUtil {
private Logger LOG;
private Context ctx;
private DumpType eventType = DumpType.EVENT_UNKNOWN;
+ private Task<? extends Serializable> openTxnTask = null;
public HiveConf getConf() {
return conf;
@@ -146,6 +147,13 @@ public class EximUtil {
this.LOG = LOG;
this.ctx = ctx;
}
+
+ public Task<? extends Serializable> getOpenTxnTask() {
+ return openTxnTask;
+ }
+ public void setOpenTxnTask(Task<? extends Serializable> openTxnTask) {
+ this.openTxnTask = openTxnTask;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
index 16ce5d5..e82a102 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hive.metastore.api.Order;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.ReplChangeManager;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.metastore.txn.TxnUtils;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.ReplCopyTask;
import org.apache.hadoop.hive.ql.exec.Task;
@@ -60,9 +61,12 @@ import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.LoadMultiFilesDesc;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc.LoadFileType;
import org.apache.hadoop.hive.ql.plan.MoveWork;
+import org.apache.hadoop.hive.ql.plan.ReplTxnWork;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.hive.ql.util.HiveStrictManagedMigration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.mapred.OutputFormat;
+import org.apache.thrift.TException;
import org.datanucleus.util.StringUtils;
import org.slf4j.Logger;
@@ -80,6 +84,7 @@ import java.util.Map;
import java.util.TreeMap;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.REPL_ENABLE_MOVE_OPTIMIZATION;
+import static org.apache.hadoop.hive.ql.util.HiveStrictManagedMigration.getHiveUpdater;
/**
* ImportSemanticAnalyzer.
@@ -190,6 +195,22 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
+ private static void upgradeTableDesc(org.apache.hadoop.hive.metastore.api.Table tableObj, MetaData rv,
+ EximUtil.SemanticAnalyzerWrapperContext x)
+ throws IOException, TException, HiveException {
+ x.getLOG().debug("Converting table " + tableObj.getTableName() + " of type " + tableObj.getTableType() +
+ " with para " + tableObj.getParameters());
+ //TODO : isPathOwnedByHive is hard coded to true, need to get it from repl dump metadata.
+ TableType tableType = TableType.valueOf(tableObj.getTableType());
+ HiveStrictManagedMigration.TableMigrationOption migrationOption =
+ HiveStrictManagedMigration.determineMigrationTypeAutomatically(tableObj, tableType,
+ null, x.getConf(), x.getHive().getMSC(), true);
+ HiveStrictManagedMigration.migrateTable(tableObj, tableType, migrationOption, false,
+ getHiveUpdater(x.getConf()), x.getHive().getMSC(), x.getConf());
+ x.getLOG().debug("Converted table " + tableObj.getTableName() + " of type " + tableObj.getTableType() +
+ " with para " + tableObj.getParameters());
+ }
+
/**
* The same code is used from both the "repl load" as well as "import".
* Given that "repl load" now supports two modes "repl load dbName [location]" and
@@ -249,7 +270,24 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
// Executed if relevant, and used to contain all the other details about the table if not.
ImportTableDesc tblDesc;
try {
- tblDesc = getBaseCreateTableDescFromTable(dbname, rv.getTable());
+ org.apache.hadoop.hive.metastore.api.Table tblObj = rv.getTable();
+ // The table can be non acid in case of replication from a cluster with STRICT_MANAGED set to false.
+ if (!TxnUtils.isTransactionalTable(tblObj) && replicationSpec.isInReplicationScope() &&
+ x.getConf().getBoolVar(HiveConf.ConfVars.HIVE_STRICT_MANAGED_TABLES) &&
+ (TableType.valueOf(tblObj.getTableType()) == TableType.MANAGED_TABLE)) {
+ //TODO : dump metadata should be read to make sure that migration is required.
+ upgradeTableDesc(tblObj, rv, x);
+ //if the conversion is from non transactional to transactional table
+ if (TxnUtils.isTransactionalTable(tblObj)) {
+ replicationSpec.setMigratingToTxnTable();
+ }
+ tblDesc = getBaseCreateTableDescFromTable(dbname, tblObj);
+ if (TableType.valueOf(tblObj.getTableType()) == TableType.EXTERNAL_TABLE) {
+ tblDesc.setExternal(true);
+ }
+ } else {
+ tblDesc = getBaseCreateTableDescFromTable(dbname, tblObj);
+ }
} catch (Exception e) {
throw new HiveException(e);
}
@@ -401,7 +439,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
boolean isAutoPurge;
boolean needRecycle;
- if (replicationSpec.isInReplicationScope() &&
+ if (replicationSpec.isInReplicationScope() && !replicationSpec.isMigratingToTxnTable() &&
x.getCtx().getConf().getBoolean(REPL_ENABLE_MOVE_OPTIMIZATION.varname, false)) {
lft = LoadFileType.IGNORE;
destPath = loadPath = tgtPath;
@@ -430,7 +468,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
lft = LoadFileType.KEEP_EXISTING;
} else {
destPath = loadPath = x.getCtx().getExternalTmpPath(tgtPath);
- lft = replace ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING;
+ lft = replace ? LoadFileType.REPLACE_ALL :
+ replicationSpec.isMigratingToTxnTable() ? LoadFileType.KEEP_EXISTING : LoadFileType.OVERWRITE_EXISTING;
}
needRecycle = false;
isAutoPurge = false;
@@ -457,7 +496,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
MoveWork moveWork = new MoveWork(x.getInputs(), x.getOutputs(), null, null, false);
- if (replicationSpec.isInReplicationScope() && AcidUtils.isTransactionalTable(table)) {
+ if (replicationSpec.isInReplicationScope() && AcidUtils.isTransactionalTable(table) &&
+ !replicationSpec.isMigratingToTxnTable()) {
LoadMultiFilesDesc loadFilesWork = new LoadMultiFilesDesc(
Collections.singletonList(destPath),
Collections.singletonList(tgtPath),
@@ -467,6 +507,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
} else {
LoadTableDesc loadTableWork = new LoadTableDesc(
loadPath, Utilities.getTableDesc(table), new TreeMap<>(), lft, writeId);
+ if (replicationSpec.isMigratingToTxnTable()) {
+ loadTableWork.setInsertOverwrite(replace);
+ }
loadTableWork.setStmtId(stmtId);
moveWork.setLoadTableWork(loadTableWork);
}
@@ -543,7 +586,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
LoadFileType loadFileType;
Path destPath;
- if (replicationSpec.isInReplicationScope() &&
+ if (replicationSpec.isInReplicationScope() && !replicationSpec.isMigratingToTxnTable() &&
x.getCtx().getConf().getBoolean(REPL_ENABLE_MOVE_OPTIMIZATION.varname, false)) {
loadFileType = LoadFileType.IGNORE;
destPath = tgtLocation;
@@ -555,7 +598,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
needRecycle = db != null && ReplChangeManager.isSourceOfReplication(db);
}
} else {
- loadFileType = replicationSpec.isReplace() ? LoadFileType.REPLACE_ALL : LoadFileType.OVERWRITE_EXISTING;
+ loadFileType = replicationSpec.isReplace() ?
+ LoadFileType.REPLACE_ALL :
+ replicationSpec.isMigratingToTxnTable() ? LoadFileType.KEEP_EXISTING : LoadFileType.OVERWRITE_EXISTING;
//Replication scope the write id will be invalid
Boolean useStagingDirectory = !AcidUtils.isTransactionalTable(table.getParameters()) ||
replicationSpec.isInReplicationScope();
@@ -565,7 +610,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
needRecycle = false;
}
- Path moveTaskSrc = !AcidUtils.isTransactionalTable(table.getParameters()) ? destPath : tgtLocation;
+ Path moveTaskSrc = !AcidUtils.isTransactionalTable(table.getParameters()) ||
+ replicationSpec.isInReplicationScope() ? destPath : tgtLocation;
if (Utilities.FILE_OP_LOGGER.isTraceEnabled()) {
Utilities.FILE_OP_LOGGER.trace("adding import work for partition with source location: "
+ srcLocation + "; target: " + tgtLocation + "; copy dest " + destPath + "; mm "
@@ -592,7 +638,8 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
// Note: this sets LoadFileType incorrectly for ACID; is that relevant for import?
// See setLoadFileType and setIsAcidIow calls elsewhere for an example.
- if (replicationSpec.isInReplicationScope() && AcidUtils.isTransactionalTable(tblDesc.getTblProps())) {
+ if (replicationSpec.isInReplicationScope() && AcidUtils.isTransactionalTable(tblDesc.getTblProps()) &&
+ !replicationSpec.isMigratingToTxnTable()) {
LoadMultiFilesDesc loadFilesWork = new LoadMultiFilesDesc(
Collections.singletonList(destPath),
Collections.singletonList(tgtLocation),
@@ -604,6 +651,9 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
partSpec.getPartSpec(),
loadFileType,
writeId);
+ if (replicationSpec.isMigratingToTxnTable()) {
+ loadTableWork.setInsertOverwrite(replicationSpec.isReplace());
+ }
loadTableWork.setStmtId(stmtId);
loadTableWork.setInheritTableSpecs(false);
moveWork.setLoadTableWork(loadTableWork);
@@ -1057,6 +1107,11 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
tblDesc.getDatabaseName(),
tblDesc.getTableName(),
null);
+ if (replicationSpec.isMigratingToTxnTable()) {
+ x.setOpenTxnTask(TaskFactory.get(new ReplTxnWork(tblDesc.getDatabaseName(),
+ tblDesc.getTableName(), ReplTxnWork.OperationType.REPL_MIGRATION_OPEN_TXN), x.getConf()));
+ updatedMetadata.setNeedCommitTxn(true);
+ }
}
if (tblDesc.getLocation() == null) {
@@ -1100,7 +1155,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer {
}
} else {
x.getLOG().debug("adding dependent CopyWork/MoveWork for table");
- t.addDependentTask(loadTable(fromURI, table, true, new Path(tblDesc.getLocation()), replicationSpec, x, writeId, stmtId));
+ t.addDependentTask(loadTable(fromURI, table, replicationSpec.isReplace(), new Path(tblDesc.getLocation()), replicationSpec, x, writeId, stmtId));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index f1fcd6e..1ebbb82 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -22,6 +22,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.ValidTxnList;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.ReplChangeManager;
import org.apache.hadoop.hive.metastore.Warehouse;
@@ -95,6 +96,9 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
public void analyzeInternal(ASTNode ast) throws SemanticException {
LOG.debug("ReplicationSemanticAanalyzer: analyzeInternal");
LOG.debug(ast.getName() + ":" + ast.getToken().getText() + "=" + ast.getText());
+ // Some of the txn related configs were not set when ReplicationSemanticAnalyzer.conf was initialized.
+ // It should be set first.
+ setTxnConfigs();
switch (ast.getToken().getType()) {
case TOK_REPL_DUMP: {
LOG.debug("ReplicationSemanticAnalyzer: analyzeInternal: dump");
@@ -124,6 +128,13 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
}
}
+ private void setTxnConfigs() {
+ String validTxnList = queryState.getConf().get(ValidTxnList.VALID_TXNS_KEY);
+ if (validTxnList != null) {
+ conf.set(ValidTxnList.VALID_TXNS_KEY, validTxnList);
+ }
+ }
+
private void initReplDump(ASTNode ast) throws HiveException {
int numChildren = ast.getChildCount();
boolean isMetaDataOnly = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
index 3115e83..39009ce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSpec.java
@@ -47,6 +47,7 @@ public class ReplicationSpec {
//TxnIds snapshot
private String validTxnList = null;
private Type specType = Type.DEFAULT; // DEFAULT means REPL_LOAD or BOOTSTRAP_DUMP or EXPORT
+ private boolean isMigratingToTxnTable = false;
// Key definitions related to replication
public enum KEY {
@@ -402,4 +403,11 @@ public class ReplicationSpec {
return SCOPE.NO_REPL;
}
}
+
+ public boolean isMigratingToTxnTable() {
+ return isMigratingToTxnTable;
+ }
+ public void setMigratingToTxnTable() {
+ isMigratingToTxnTable = true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/UpdatedMetaDataTracker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/UpdatedMetaDataTracker.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/UpdatedMetaDataTracker.java
index 614e071..1f20698 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/UpdatedMetaDataTracker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/UpdatedMetaDataTracker.java
@@ -71,12 +71,21 @@ public class UpdatedMetaDataTracker {
private List<UpdateMetaData> updateMetaDataList;
private Map<String, Integer> updateMetaDataMap;
+ private boolean needCommitTxn = false;
public UpdatedMetaDataTracker() {
updateMetaDataList = new ArrayList<>();
updateMetaDataMap = new HashMap<>();
}
+ public void setNeedCommitTxn(boolean needCommitTxn) {
+ this.needCommitTxn = needCommitTxn;
+ }
+
+ public boolean isNeedCommitTxn() {
+ return needCommitTxn;
+ }
+
public void copyUpdatedMetadata(UpdatedMetaDataTracker other) {
int size = updateMetaDataList.size();
for (UpdateMetaData updateMetaDataOther : other.updateMetaDataList) {
@@ -93,6 +102,7 @@ public class UpdatedMetaDataTracker {
}
}
}
+ this.needCommitTxn = other.needCommitTxn;
}
public void set(String replState, String dbName, String tableName, Map <String, String> partSpec)
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
index 4a2fdd2..b95a35a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/DropPartitionHandler.java
@@ -41,8 +41,7 @@ public class DropPartitionHandler extends AbstractMessageHandler {
String actualDbName = context.isDbNameEmpty() ? msg.getDB() : context.dbName;
String actualTblName = context.isTableNameEmpty() ? msg.getTable() : context.tableName;
Map<Integer, List<ExprNodeGenericFuncDesc>> partSpecs =
- ReplUtils.genPartSpecs(new Table(msg.getTableObj()),
- msg.getPartitions());
+ ReplUtils.genPartSpecs(new Table(msg.getTableObj()), msg.getPartitions());
if (partSpecs.size() > 0) {
DropTableDesc dropPtnDesc = new DropTableDesc(actualDbName + "." + actualTblName,
partSpecs, null, true, context.eventOnlyReplicationSpec());
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
index 0035026..1125f69 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenamePartitionHandler.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hive.ql.parse.repl.load.message;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -46,10 +45,9 @@ public class RenamePartitionHandler extends AbstractMessageHandler {
Map<String, String> oldPartSpec = new LinkedHashMap<>();
String tableName = actualDbName + "." + actualTblName;
try {
- Table tblObj = msg.getTableObj();
Iterator<String> beforeIterator = msg.getPtnObjBefore().getValuesIterator();
Iterator<String> afterIterator = msg.getPtnObjAfter().getValuesIterator();
- for (FieldSchema fs : tblObj.getPartitionKeys()) {
+ for (FieldSchema fs : msg.getTableObj().getPartitionKeys()) {
oldPartSpec.put(fs.getName(), beforeIterator.next());
newPartSpec.put(fs.getName(), afterIterator.next());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
index 83433d7..ddf2ca1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/RenameTableHandler.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hive.ql.parse.repl.load.message;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
@@ -40,8 +41,10 @@ public class RenameTableHandler extends AbstractMessageHandler {
"RENAMES of tables are not supported for table-level replication");
}
try {
- String oldDbName = msg.getTableObjBefore().getDbName();
- String newDbName = msg.getTableObjAfter().getDbName();
+ Table tableObjBefore = msg.getTableObjBefore();
+ Table tableObjAfter = msg.getTableObjAfter();
+ String oldDbName = tableObjBefore.getDbName();
+ String newDbName = tableObjAfter.getDbName();
if (!context.isDbNameEmpty()) {
// If we're loading into a db, instead of into the warehouse, then the oldDbName and
@@ -56,8 +59,8 @@ public class RenameTableHandler extends AbstractMessageHandler {
}
}
- String oldName = StatsUtils.getFullyQualifiedTableName(oldDbName, msg.getTableObjBefore().getTableName());
- String newName = StatsUtils.getFullyQualifiedTableName(newDbName, msg.getTableObjAfter().getTableName());
+ String oldName = StatsUtils.getFullyQualifiedTableName(oldDbName, tableObjBefore.getTableName());
+ String newName = StatsUtils.getFullyQualifiedTableName(newDbName, tableObjAfter.getTableName());
AlterTableDesc renameTableDesc = new AlterTableDesc(
oldName, newName, false, context.eventOnlyReplicationSpec());
Task<DDLWork> renameTableTask = TaskFactory.get(
@@ -67,7 +70,7 @@ public class RenameTableHandler extends AbstractMessageHandler {
// oldDbName and newDbName *will* be the same if we're here
updatedMetadata.set(context.dmd.getEventTo().toString(), newDbName,
- msg.getTableObjAfter().getTableName(), null);
+ tableObjAfter.getTableName(), null);
// Note : edge-case here in interaction with table-level REPL LOAD, where that nukes out
// tablesUpdated. However, we explicitly don't support repl of that sort, and error out above
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
index 3fb18d8..f5f4459 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TableHandler.java
@@ -21,11 +21,14 @@ import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.parse.EximUtil;
+import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.parse.ImportSemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.ReplTxnWork;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import static org.apache.hadoop.hive.ql.parse.repl.DumpType.EVENT_ALTER_PARTITION;
@@ -59,6 +62,14 @@ public class TableHandler extends AbstractMessageHandler {
(context.precursor != null), null, context.tableName, context.dbName,
null, context.location, x, updatedMetadata, context.getTxnMgr(), writeId);
+ Task<? extends Serializable> openTxnTask = x.getOpenTxnTask();
+ if (openTxnTask != null && !importTasks.isEmpty()) {
+ for (Task<? extends Serializable> t : importTasks) {
+ openTxnTask.addDependentTask(t);
+ }
+ importTasks.add(openTxnTask);
+ }
+
return importTasks;
} catch (Exception e) {
throw new SemanticException(e);
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
index 0d5ac31..dec6ed5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncatePartitionHandler.java
@@ -21,12 +21,12 @@ import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.TruncateTableDesc;
import java.io.Serializable;
-import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -40,8 +40,9 @@ public class TruncatePartitionHandler extends AbstractMessageHandler {
String actualTblName = context.isTableNameEmpty() ? msg.getTable() : context.tableName;
Map<String, String> partSpec = new LinkedHashMap<>();
+ org.apache.hadoop.hive.metastore.api.Table tblObj;
try {
- org.apache.hadoop.hive.metastore.api.Table tblObj = msg.getTableObj();
+ tblObj = msg.getTableObj();
Iterator<String> afterIterator = msg.getPtnObjAfter().getValuesIterator();
for (FieldSchema fs : tblObj.getPartitionKeys()) {
partSpec.put(fs.getName(), afterIterator.next());
@@ -63,6 +64,12 @@ public class TruncatePartitionHandler extends AbstractMessageHandler {
context.log.debug("Added truncate ptn task : {}:{}:{}", truncatePtnTask.getId(),
truncateTableDesc.getTableName(), truncateTableDesc.getWriteId());
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, partSpec);
- return Collections.singletonList(truncatePtnTask);
+
+ try {
+ return ReplUtils.addOpenTxnTaskForMigration(actualDbName, actualTblName,
+ context.hiveConf, updatedMetadata, truncatePtnTask, tblObj);
+ } catch (Exception e) {
+ throw new SemanticException(e.getMessage());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
index d18a9e1..f037cbb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/load/message/TruncateTableHandler.java
@@ -20,12 +20,12 @@ package org.apache.hadoop.hive.ql.parse.repl.load.message;
import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.repl.util.ReplUtils;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.DDLWork;
import org.apache.hadoop.hive.ql.plan.TruncateTableDesc;
import java.io.Serializable;
-import java.util.Collections;
import java.util.List;
public class TruncateTableHandler extends AbstractMessageHandler {
@@ -45,6 +45,12 @@ public class TruncateTableHandler extends AbstractMessageHandler {
context.log.debug("Added truncate tbl task : {}:{}:{}", truncateTableTask.getId(),
truncateTableDesc.getTableName(), truncateTableDesc.getWriteId());
updatedMetadata.set(context.dmd.getEventTo().toString(), actualDbName, actualTblName, null);
- return Collections.singletonList(truncateTableTask);
+
+ try {
+ return ReplUtils.addOpenTxnTaskForMigration(actualDbName, actualTblName,
+ context.hiveConf, updatedMetadata, truncateTableTask, msg.getTableObjBefore());
+ } catch (Exception e) {
+ throw new SemanticException(e.getMessage());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
index d9333b5..bed0581 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/LoadTableDesc.java
@@ -253,6 +253,10 @@ public class LoadTableDesc extends LoadDesc implements Serializable {
return currentWriteId == null ? 0 : currentWriteId;
}
+ public void setWriteId(long writeId) {
+ currentWriteId = writeId;
+ }
+
public int getStmtId() {
return stmtId;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/plan/ReplTxnWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReplTxnWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReplTxnWork.java
index a6ab836..a9f98cc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReplTxnWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReplTxnWork.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.plan;
import java.io.Serializable;
-
+import org.apache.hadoop.hive.metastore.api.ReplLastIdInfo;
import org.apache.hadoop.hive.metastore.api.TxnToWriteId;
import org.apache.hadoop.hive.metastore.api.WriteEventInfo;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
@@ -43,13 +43,15 @@ public class ReplTxnWork implements Serializable {
private List<TxnToWriteId> txnToWriteIdList;
private ReplicationSpec replicationSpec;
private List<WriteEventInfo> writeEventInfos;
+ private ReplLastIdInfo replLastIdInfo;
/**
* OperationType.
* Different kind of events supported for replaying.
*/
public enum OperationType {
- REPL_OPEN_TXN, REPL_ABORT_TXN, REPL_COMMIT_TXN, REPL_ALLOC_WRITE_ID, REPL_WRITEID_STATE
+ REPL_OPEN_TXN, REPL_ABORT_TXN, REPL_COMMIT_TXN, REPL_ALLOC_WRITE_ID, REPL_WRITEID_STATE,
+ REPL_MIGRATION_OPEN_TXN, REPL_MIGRATION_COMMIT_TXN
}
OperationType operation;
@@ -64,6 +66,7 @@ public class ReplTxnWork implements Serializable {
this.txnToWriteIdList = txnToWriteIdList;
this.replicationSpec = replicationSpec;
this.writeEventInfos = null;
+ this.replLastIdInfo = null;
}
public ReplTxnWork(String replPolicy, String dbName, String tableName, List<Long> txnIds, OperationType type,
@@ -90,6 +93,17 @@ public class ReplTxnWork implements Serializable {
this.operation = type;
}
+ public ReplTxnWork(String dbName, String tableName, OperationType type) {
+ this(null, dbName, tableName, null, type, null, null);
+ assert type == OperationType.REPL_MIGRATION_OPEN_TXN;
+ }
+
+ public ReplTxnWork(ReplLastIdInfo replLastIdInfo, OperationType type) {
+ this(null, null, null, null, type, null, null);
+ assert type == OperationType.REPL_MIGRATION_COMMIT_TXN;
+ this.replLastIdInfo = replLastIdInfo;
+ }
+
public void addWriteEventInfo(WriteEventInfo writeEventInfo) {
if (this.writeEventInfos == null) {
this.writeEventInfos = new ArrayList<>();
@@ -136,4 +150,8 @@ public class ReplTxnWork implements Serializable {
public List<WriteEventInfo> getWriteEventInfos() {
return writeEventInfos;
}
+
+ public ReplLastIdInfo getReplLastIdInfo() {
+ return replLastIdInfo;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
index 13a8af7..9535bed 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
@@ -77,7 +77,7 @@ public class HiveStrictManagedMigration {
private static final Logger LOG = LoggerFactory.getLogger(HiveStrictManagedMigration.class);
- enum TableMigrationOption {
+ public enum TableMigrationOption {
NONE, // Do nothing
VALIDATE, // No migration, just validate that the tables
AUTOMATIC, // Automatically determine if the table should be managed or external
@@ -428,6 +428,32 @@ public class HiveStrictManagedMigration {
}
}
+ public static boolean migrateTable(Table tableObj, TableType tableType, TableMigrationOption migrationOption,
+ boolean dryRun, HiveUpdater hiveUpdater, IMetaStoreClient hms, Configuration conf)
+ throws HiveException, IOException, TException {
+ switch (migrationOption) {
+ case EXTERNAL:
+ migrateToExternalTable(tableObj, tableType, dryRun, hiveUpdater);
+ break;
+ case MANAGED:
+ migrateToManagedTable(tableObj, tableType, dryRun, hiveUpdater, hms, conf);
+ break;
+ case NONE:
+ break;
+ case VALIDATE:
+ // Check that the table is valid under strict managed tables mode.
+ String reason = HiveStrictManagedUtils.validateStrictManagedTable(conf, tableObj);
+ if (reason != null) {
+ LOG.warn(reason);
+ return true;
+ }
+ break;
+ default:
+ throw new IllegalArgumentException("Unexpected table migration option " + migrationOption);
+ }
+ return false;
+ }
+
void processTable(Database dbObj, String tableName, boolean modifyDefaultManagedLocation)
throws HiveException, IOException, TException {
String dbName = dbObj.getName();
@@ -435,41 +461,16 @@ public class HiveStrictManagedMigration {
Table tableObj = hms.getTable(dbName, tableName);
TableType tableType = TableType.valueOf(tableObj.getTableType());
- boolean tableMigrated;
TableMigrationOption migrationOption = runOptions.migrationOption;
if (migrationOption == TableMigrationOption.AUTOMATIC) {
- migrationOption = determineMigrationTypeAutomatically(tableObj, tableType);
+ migrationOption = determineMigrationTypeAutomatically(tableObj, tableType, ownerName, conf, hms, null);
}
- switch (migrationOption) {
- case EXTERNAL:
- tableMigrated = migrateToExternalTable(tableObj, tableType);
- if (tableMigrated) {
- tableType = TableType.EXTERNAL_TABLE;
- }
- break;
- case MANAGED:
- tableMigrated = migrateToManagedTable(tableObj, tableType);
- if (tableMigrated) {
- tableType = TableType.MANAGED_TABLE;
- }
- break;
- case NONE:
- break;
- case VALIDATE:
- // Check that the table is valid under strict managed tables mode.
- String reason = HiveStrictManagedUtils.validateStrictManagedTable(conf, tableObj);
- if (reason != null) {
- LOG.warn(reason);
- failedValidationChecks = true;
- }
- break;
- default:
- throw new IllegalArgumentException("Unexpected table migration option " + runOptions.migrationOption);
- }
+ failedValidationChecks = migrateTable(tableObj, tableType, migrationOption, runOptions.dryRun,
+ getHiveUpdater(), hms, conf);
- if (tableType == TableType.MANAGED_TABLE) {
+ if (!failedValidationChecks && (TableType.valueOf(tableObj.getTableType()) == TableType.MANAGED_TABLE)) {
Path tablePath = new Path(tableObj.getSd().getLocation());
if (modifyDefaultManagedLocation && shouldModifyTableLocation(dbObj, tableObj)) {
Path newTablePath = wh.getDnsPath(
@@ -623,7 +624,8 @@ public class HiveStrictManagedMigration {
}
}
- void renameFilesToConformToAcid(Table tableObj) throws IOException, TException {
+ static void renameFilesToConformToAcid(Table tableObj, IMetaStoreClient hms, Configuration conf, boolean dryRun)
+ throws IOException, TException {
if (isPartitionedTable(tableObj)) {
String dbName = tableObj.getDbName();
String tableName = tableObj.getTableName();
@@ -634,7 +636,7 @@ public class HiveStrictManagedMigration {
FileSystem fs = partPath.getFileSystem(conf);
if (fs.exists(partPath)) {
UpgradeTool.handleRenameFiles(tableObj, partPath,
- !runOptions.dryRun, conf, tableObj.getSd().getBucketColsSize() > 0, null);
+ !dryRun, conf, tableObj.getSd().getBucketColsSize() > 0, null);
}
}
} else {
@@ -642,12 +644,13 @@ public class HiveStrictManagedMigration {
FileSystem fs = tablePath.getFileSystem(conf);
if (fs.exists(tablePath)) {
UpgradeTool.handleRenameFiles(tableObj, tablePath,
- !runOptions.dryRun, conf, tableObj.getSd().getBucketColsSize() > 0, null);
+ !dryRun, conf, tableObj.getSd().getBucketColsSize() > 0, null);
}
}
}
- TableMigrationOption determineMigrationTypeAutomatically(Table tableObj, TableType tableType)
+ public static TableMigrationOption determineMigrationTypeAutomatically(Table tableObj, TableType tableType,
+ String ownerName, Configuration conf, IMetaStoreClient hms, Boolean isPathOwnedByHive)
throws IOException, MetaException, TException {
TableMigrationOption result = TableMigrationOption.NONE;
String msg;
@@ -657,7 +660,7 @@ public class HiveStrictManagedMigration {
// Always keep transactional tables as managed tables.
result = TableMigrationOption.MANAGED;
} else {
- String reason = shouldTableBeExternal(tableObj);
+ String reason = shouldTableBeExternal(tableObj, ownerName, conf, hms, isPathOwnedByHive);
if (reason != null) {
LOG.debug("Converting {} to external table. {}", getQualifiedName(tableObj), reason);
result = TableMigrationOption.EXTERNAL;
@@ -697,7 +700,8 @@ public class HiveStrictManagedMigration {
convertToMMTableProps.put("transactional_properties", "insert_only");
}
- boolean migrateToExternalTable(Table tableObj, TableType tableType) throws HiveException {
+ static boolean migrateToExternalTable(Table tableObj, TableType tableType, boolean dryRun, HiveUpdater hiveUpdater)
+ throws HiveException {
String msg;
switch (tableType) {
case MANAGED_TABLE:
@@ -708,9 +712,9 @@ public class HiveStrictManagedMigration {
return false;
}
LOG.info("Converting {} to external table ...", getQualifiedName(tableObj));
- if (!runOptions.dryRun) {
+ if (!dryRun) {
tableObj.setTableType(TableType.EXTERNAL_TABLE.toString());
- getHiveUpdater().updateTableProperties(tableObj, convertToExternalTableProps);
+ hiveUpdater.updateTableProperties(tableObj, convertToExternalTableProps);
}
return true;
case EXTERNAL_TABLE:
@@ -727,13 +731,13 @@ public class HiveStrictManagedMigration {
return false;
}
- boolean canTableBeFullAcid(Table tableObj) throws MetaException {
+ static boolean canTableBeFullAcid(Table tableObj) throws MetaException {
// Table must be acid-compatible table format, and no sorting columns.
return TransactionalValidationListener.conformToAcid(tableObj) &&
(tableObj.getSd().getSortColsSize() <= 0);
}
- Map<String, String> getTablePropsForConversionToTransactional(Map<String, String> props,
+ static Map<String, String> getTablePropsForConversionToTransactional(Map<String, String> props,
boolean convertFromExternal) {
if (convertFromExternal) {
// Copy the properties to a new map so we can add EXTERNAL=FALSE
@@ -743,7 +747,9 @@ public class HiveStrictManagedMigration {
return props;
}
- boolean migrateToManagedTable(Table tableObj, TableType tableType) throws HiveException, IOException, MetaException, TException {
+ static boolean migrateToManagedTable(Table tableObj, TableType tableType, boolean dryRun, HiveUpdater hiveUpdater,
+ IMetaStoreClient hms, Configuration conf)
+ throws HiveException, IOException, MetaException, TException {
boolean convertFromExternal = false;
switch (tableType) {
@@ -784,20 +790,22 @@ public class HiveStrictManagedMigration {
// TODO: option to allow converting ORC file to insert-only transactional?
LOG.info("Converting {} to full transactional table", getQualifiedName(tableObj));
- renameFilesToConformToAcid(tableObj);
+ if (hiveUpdater.doFileRename) {
+ renameFilesToConformToAcid(tableObj, hms, conf, dryRun);
+ }
- if (!runOptions.dryRun) {
+ if (!dryRun) {
Map<String, String> props = getTablePropsForConversionToTransactional(
convertToAcidTableProps, convertFromExternal);
- getHiveUpdater().updateTableProperties(tableObj, props);
+ hiveUpdater.updateTableProperties(tableObj, props);
}
return true;
} else {
LOG.info("Converting {} to insert-only transactional table", getQualifiedName(tableObj));
- if (!runOptions.dryRun) {
+ if (!dryRun) {
Map<String, String> props = getTablePropsForConversionToTransactional(
convertToMMTableProps, convertFromExternal);
- getHiveUpdater().updateTableProperties(tableObj, props);
+ hiveUpdater.updateTableProperties(tableObj, props);
}
return true;
}
@@ -809,7 +817,9 @@ public class HiveStrictManagedMigration {
}
}
- String shouldTableBeExternal(Table tableObj) throws IOException, MetaException, TException {
+ static String shouldTableBeExternal(Table tableObj, String ownerName, Configuration conf,
+ IMetaStoreClient hms, Boolean isPathOwnedByHive)
+ throws IOException, MetaException, TException {
if (MetaStoreUtils.isNonNativeTable(tableObj)) {
return "Table is a non-native (StorageHandler) table";
}
@@ -824,14 +834,19 @@ public class HiveStrictManagedMigration {
// then assume table is using storage-based auth - set external.
// Transactional tables should still remain transactional,
// but we should have already checked for that before this point.
- if (shouldTablePathBeExternal(tableObj, ownerName)) {
+ if (isPathOwnedByHive != null) {
+ // for replication flow, the path ownership must be verified at source cluster itself.
+ return isPathOwnedByHive ? null :
+ String.format("One or more table directories is not owned by hive or non-HDFS path at source cluster");
+ } else if (shouldTablePathBeExternal(tableObj, ownerName, conf, hms)) {
return String.format("One or more table directories not owned by %s, or non-HDFS path", ownerName);
}
return null;
}
- boolean shouldTablePathBeExternal(Table tableObj, String userName) throws IOException, MetaException, TException {
+ static boolean shouldTablePathBeExternal(Table tableObj, String ownerName, Configuration conf, IMetaStoreClient hms)
+ throws IOException, TException {
boolean shouldBeExternal = false;
String dbName = tableObj.getDbName();
String tableName = tableObj.getTableName();
@@ -876,9 +891,13 @@ public class HiveStrictManagedMigration {
}
}
+ public static HiveUpdater getHiveUpdater(HiveConf conf) throws HiveException {
+ return new HiveUpdater(conf, false);
+ }
+
HiveUpdater getHiveUpdater() throws HiveException {
if (hiveUpdater == null) {
- hiveUpdater = new HiveUpdater();
+ hiveUpdater = new HiveUpdater(conf, true);
}
return hiveUpdater;
}
@@ -895,12 +914,14 @@ public class HiveStrictManagedMigration {
}
}
- class HiveUpdater {
+ public static class HiveUpdater {
Hive hive;
+ boolean doFileRename;
- HiveUpdater() throws HiveException {
+ HiveUpdater(HiveConf conf, boolean fileRename) throws HiveException {
hive = Hive.get(conf);
Hive.set(hive);
+ doFileRename = fileRename;
}
void close() {
@@ -1041,15 +1062,19 @@ public class HiveStrictManagedMigration {
void updateTableProperties(Table table, Map<String, String> props) throws HiveException {
StringBuilder sb = new StringBuilder();
boolean isTxn = TxnUtils.isTransactionalTable(table);
- org.apache.hadoop.hive.ql.metadata.Table modifiedTable =
- new org.apache.hadoop.hive.ql.metadata.Table(table);
+ org.apache.hadoop.hive.ql.metadata.Table modifiedTable = doFileRename ?
+ new org.apache.hadoop.hive.ql.metadata.Table(table) : null;
if (props.size() == 0) {
return;
}
boolean first = true;
for (String key : props.keySet()) {
String value = props.get(key);
- modifiedTable.getParameters().put(key, value);
+ if (modifiedTable == null) {
+ table.getParameters().put(key, value);
+ } else {
+ modifiedTable.getParameters().put(key, value);
+ }
// Build properties list for logging
if (first) {
@@ -1069,7 +1094,9 @@ public class HiveStrictManagedMigration {
// Note: for now, this is always called to convert the table to either external, or ACID/MM,
// so the original table would be non-txn and the transaction wouldn't be opened.
- alterTableInternal(isTxn, table, modifiedTable);
+ if (modifiedTable != null) {
+ alterTableInternal(isTxn, table, modifiedTable);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddDynamicPartitions.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddDynamicPartitions.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddDynamicPartitions.java
index 45618e7..9c33229 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddDynamicPartitions.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AddDynamicPartitions.java
@@ -816,13 +816,13 @@ import org.slf4j.LoggerFactory;
case 5: // PARTITIONNAMES
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list716 = iprot.readListBegin();
- struct.partitionnames = new ArrayList<String>(_list716.size);
- String _elem717;
- for (int _i718 = 0; _i718 < _list716.size; ++_i718)
+ org.apache.thrift.protocol.TList _list724 = iprot.readListBegin();
+ struct.partitionnames = new ArrayList<String>(_list724.size);
+ String _elem725;
+ for (int _i726 = 0; _i726 < _list724.size; ++_i726)
{
- _elem717 = iprot.readString();
- struct.partitionnames.add(_elem717);
+ _elem725 = iprot.readString();
+ struct.partitionnames.add(_elem725);
}
iprot.readListEnd();
}
@@ -872,9 +872,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(PARTITIONNAMES_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, struct.partitionnames.size()));
- for (String _iter719 : struct.partitionnames)
+ for (String _iter727 : struct.partitionnames)
{
- oprot.writeString(_iter719);
+ oprot.writeString(_iter727);
}
oprot.writeListEnd();
}
@@ -910,9 +910,9 @@ import org.slf4j.LoggerFactory;
oprot.writeString(struct.tablename);
{
oprot.writeI32(struct.partitionnames.size());
- for (String _iter720 : struct.partitionnames)
+ for (String _iter728 : struct.partitionnames)
{
- oprot.writeString(_iter720);
+ oprot.writeString(_iter728);
}
}
BitSet optionals = new BitSet();
@@ -937,13 +937,13 @@ import org.slf4j.LoggerFactory;
struct.tablename = iprot.readString();
struct.setTablenameIsSet(true);
{
- org.apache.thrift.protocol.TList _list721 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
- struct.partitionnames = new ArrayList<String>(_list721.size);
- String _elem722;
- for (int _i723 = 0; _i723 < _list721.size; ++_i723)
+ org.apache.thrift.protocol.TList _list729 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING, iprot.readI32());
+ struct.partitionnames = new ArrayList<String>(_list729.size);
+ String _elem730;
+ for (int _i731 = 0; _i731 < _list729.size; ++_i731)
{
- _elem722 = iprot.readString();
- struct.partitionnames.add(_elem722);
+ _elem730 = iprot.readString();
+ struct.partitionnames.add(_elem730);
}
}
struct.setPartitionnamesIsSet(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsRequest.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsRequest.java
index 5fcb98f..d05e7ba 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsRequest.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsRequest.java
@@ -716,13 +716,13 @@ import org.slf4j.LoggerFactory;
case 3: // TXN_IDS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list642 = iprot.readListBegin();
- struct.txnIds = new ArrayList<Long>(_list642.size);
- long _elem643;
- for (int _i644 = 0; _i644 < _list642.size; ++_i644)
+ org.apache.thrift.protocol.TList _list650 = iprot.readListBegin();
+ struct.txnIds = new ArrayList<Long>(_list650.size);
+ long _elem651;
+ for (int _i652 = 0; _i652 < _list650.size; ++_i652)
{
- _elem643 = iprot.readI64();
- struct.txnIds.add(_elem643);
+ _elem651 = iprot.readI64();
+ struct.txnIds.add(_elem651);
}
iprot.readListEnd();
}
@@ -742,14 +742,14 @@ import org.slf4j.LoggerFactory;
case 5: // SRC_TXN_TO_WRITE_ID_LIST
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list645 = iprot.readListBegin();
- struct.srcTxnToWriteIdList = new ArrayList<TxnToWriteId>(_list645.size);
- TxnToWriteId _elem646;
- for (int _i647 = 0; _i647 < _list645.size; ++_i647)
+ org.apache.thrift.protocol.TList _list653 = iprot.readListBegin();
+ struct.srcTxnToWriteIdList = new ArrayList<TxnToWriteId>(_list653.size);
+ TxnToWriteId _elem654;
+ for (int _i655 = 0; _i655 < _list653.size; ++_i655)
{
- _elem646 = new TxnToWriteId();
- _elem646.read(iprot);
- struct.srcTxnToWriteIdList.add(_elem646);
+ _elem654 = new TxnToWriteId();
+ _elem654.read(iprot);
+ struct.srcTxnToWriteIdList.add(_elem654);
}
iprot.readListEnd();
}
@@ -786,9 +786,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(TXN_IDS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.txnIds.size()));
- for (long _iter648 : struct.txnIds)
+ for (long _iter656 : struct.txnIds)
{
- oprot.writeI64(_iter648);
+ oprot.writeI64(_iter656);
}
oprot.writeListEnd();
}
@@ -807,9 +807,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(SRC_TXN_TO_WRITE_ID_LIST_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.srcTxnToWriteIdList.size()));
- for (TxnToWriteId _iter649 : struct.srcTxnToWriteIdList)
+ for (TxnToWriteId _iter657 : struct.srcTxnToWriteIdList)
{
- _iter649.write(oprot);
+ _iter657.write(oprot);
}
oprot.writeListEnd();
}
@@ -849,9 +849,9 @@ import org.slf4j.LoggerFactory;
if (struct.isSetTxnIds()) {
{
oprot.writeI32(struct.txnIds.size());
- for (long _iter650 : struct.txnIds)
+ for (long _iter658 : struct.txnIds)
{
- oprot.writeI64(_iter650);
+ oprot.writeI64(_iter658);
}
}
}
@@ -861,9 +861,9 @@ import org.slf4j.LoggerFactory;
if (struct.isSetSrcTxnToWriteIdList()) {
{
oprot.writeI32(struct.srcTxnToWriteIdList.size());
- for (TxnToWriteId _iter651 : struct.srcTxnToWriteIdList)
+ for (TxnToWriteId _iter659 : struct.srcTxnToWriteIdList)
{
- _iter651.write(oprot);
+ _iter659.write(oprot);
}
}
}
@@ -879,13 +879,13 @@ import org.slf4j.LoggerFactory;
BitSet incoming = iprot.readBitSet(3);
if (incoming.get(0)) {
{
- org.apache.thrift.protocol.TList _list652 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
- struct.txnIds = new ArrayList<Long>(_list652.size);
- long _elem653;
- for (int _i654 = 0; _i654 < _list652.size; ++_i654)
+ org.apache.thrift.protocol.TList _list660 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
+ struct.txnIds = new ArrayList<Long>(_list660.size);
+ long _elem661;
+ for (int _i662 = 0; _i662 < _list660.size; ++_i662)
{
- _elem653 = iprot.readI64();
- struct.txnIds.add(_elem653);
+ _elem661 = iprot.readI64();
+ struct.txnIds.add(_elem661);
}
}
struct.setTxnIdsIsSet(true);
@@ -896,14 +896,14 @@ import org.slf4j.LoggerFactory;
}
if (incoming.get(2)) {
{
- org.apache.thrift.protocol.TList _list655 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.srcTxnToWriteIdList = new ArrayList<TxnToWriteId>(_list655.size);
- TxnToWriteId _elem656;
- for (int _i657 = 0; _i657 < _list655.size; ++_i657)
+ org.apache.thrift.protocol.TList _list663 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ struct.srcTxnToWriteIdList = new ArrayList<TxnToWriteId>(_list663.size);
+ TxnToWriteId _elem664;
+ for (int _i665 = 0; _i665 < _list663.size; ++_i665)
{
- _elem656 = new TxnToWriteId();
- _elem656.read(iprot);
- struct.srcTxnToWriteIdList.add(_elem656);
+ _elem664 = new TxnToWriteId();
+ _elem664.read(iprot);
+ struct.srcTxnToWriteIdList.add(_elem664);
}
}
struct.setSrcTxnToWriteIdListIsSet(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsResponse.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsResponse.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsResponse.java
index 2a13eba..193179b 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsResponse.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AllocateTableWriteIdsResponse.java
@@ -354,14 +354,14 @@ import org.slf4j.LoggerFactory;
case 1: // TXN_TO_WRITE_IDS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list658 = iprot.readListBegin();
- struct.txnToWriteIds = new ArrayList<TxnToWriteId>(_list658.size);
- TxnToWriteId _elem659;
- for (int _i660 = 0; _i660 < _list658.size; ++_i660)
+ org.apache.thrift.protocol.TList _list666 = iprot.readListBegin();
+ struct.txnToWriteIds = new ArrayList<TxnToWriteId>(_list666.size);
+ TxnToWriteId _elem667;
+ for (int _i668 = 0; _i668 < _list666.size; ++_i668)
{
- _elem659 = new TxnToWriteId();
- _elem659.read(iprot);
- struct.txnToWriteIds.add(_elem659);
+ _elem667 = new TxnToWriteId();
+ _elem667.read(iprot);
+ struct.txnToWriteIds.add(_elem667);
}
iprot.readListEnd();
}
@@ -387,9 +387,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(TXN_TO_WRITE_IDS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.txnToWriteIds.size()));
- for (TxnToWriteId _iter661 : struct.txnToWriteIds)
+ for (TxnToWriteId _iter669 : struct.txnToWriteIds)
{
- _iter661.write(oprot);
+ _iter669.write(oprot);
}
oprot.writeListEnd();
}
@@ -414,9 +414,9 @@ import org.slf4j.LoggerFactory;
TTupleProtocol oprot = (TTupleProtocol) prot;
{
oprot.writeI32(struct.txnToWriteIds.size());
- for (TxnToWriteId _iter662 : struct.txnToWriteIds)
+ for (TxnToWriteId _iter670 : struct.txnToWriteIds)
{
- _iter662.write(oprot);
+ _iter670.write(oprot);
}
}
}
@@ -425,14 +425,14 @@ import org.slf4j.LoggerFactory;
public void read(org.apache.thrift.protocol.TProtocol prot, AllocateTableWriteIdsResponse struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
{
- org.apache.thrift.protocol.TList _list663 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.txnToWriteIds = new ArrayList<TxnToWriteId>(_list663.size);
- TxnToWriteId _elem664;
- for (int _i665 = 0; _i665 < _list663.size; ++_i665)
+ org.apache.thrift.protocol.TList _list671 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ struct.txnToWriteIds = new ArrayList<TxnToWriteId>(_list671.size);
+ TxnToWriteId _elem672;
+ for (int _i673 = 0; _i673 < _list671.size; ++_i673)
{
- _elem664 = new TxnToWriteId();
- _elem664.read(iprot);
- struct.txnToWriteIds.add(_elem664);
+ _elem672 = new TxnToWriteId();
+ _elem672.read(iprot);
+ struct.txnToWriteIds.add(_elem672);
}
}
struct.setTxnToWriteIdsIsSet(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AlterPartitionsRequest.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AlterPartitionsRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AlterPartitionsRequest.java
index 4d4595a..d85dda5 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AlterPartitionsRequest.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/AlterPartitionsRequest.java
@@ -877,14 +877,14 @@ import org.slf4j.LoggerFactory;
case 4: // PARTITIONS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list952 = iprot.readListBegin();
- struct.partitions = new ArrayList<Partition>(_list952.size);
- Partition _elem953;
- for (int _i954 = 0; _i954 < _list952.size; ++_i954)
+ org.apache.thrift.protocol.TList _list960 = iprot.readListBegin();
+ struct.partitions = new ArrayList<Partition>(_list960.size);
+ Partition _elem961;
+ for (int _i962 = 0; _i962 < _list960.size; ++_i962)
{
- _elem953 = new Partition();
- _elem953.read(iprot);
- struct.partitions.add(_elem953);
+ _elem961 = new Partition();
+ _elem961.read(iprot);
+ struct.partitions.add(_elem961);
}
iprot.readListEnd();
}
@@ -952,9 +952,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(PARTITIONS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, struct.partitions.size()));
- for (Partition _iter955 : struct.partitions)
+ for (Partition _iter963 : struct.partitions)
{
- _iter955.write(oprot);
+ _iter963.write(oprot);
}
oprot.writeListEnd();
}
@@ -1000,9 +1000,9 @@ import org.slf4j.LoggerFactory;
oprot.writeString(struct.tableName);
{
oprot.writeI32(struct.partitions.size());
- for (Partition _iter956 : struct.partitions)
+ for (Partition _iter964 : struct.partitions)
{
- _iter956.write(oprot);
+ _iter964.write(oprot);
}
}
BitSet optionals = new BitSet();
@@ -1041,14 +1041,14 @@ import org.slf4j.LoggerFactory;
struct.tableName = iprot.readString();
struct.setTableNameIsSet(true);
{
- org.apache.thrift.protocol.TList _list957 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
- struct.partitions = new ArrayList<Partition>(_list957.size);
- Partition _elem958;
- for (int _i959 = 0; _i959 < _list957.size; ++_i959)
+ org.apache.thrift.protocol.TList _list965 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRUCT, iprot.readI32());
+ struct.partitions = new ArrayList<Partition>(_list965.size);
+ Partition _elem966;
+ for (int _i967 = 0; _i967 < _list965.size; ++_i967)
{
- _elem958 = new Partition();
- _elem958.read(iprot);
- struct.partitions.add(_elem958);
+ _elem966 = new Partition();
+ _elem966.read(iprot);
+ struct.partitions.add(_elem966);
}
}
struct.setPartitionsIsSet(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClearFileMetadataRequest.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClearFileMetadataRequest.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClearFileMetadataRequest.java
index 3fdd295..3eb55b1 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClearFileMetadataRequest.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClearFileMetadataRequest.java
@@ -351,13 +351,13 @@ import org.slf4j.LoggerFactory;
case 1: // FILE_IDS
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list832 = iprot.readListBegin();
- struct.fileIds = new ArrayList<Long>(_list832.size);
- long _elem833;
- for (int _i834 = 0; _i834 < _list832.size; ++_i834)
+ org.apache.thrift.protocol.TList _list840 = iprot.readListBegin();
+ struct.fileIds = new ArrayList<Long>(_list840.size);
+ long _elem841;
+ for (int _i842 = 0; _i842 < _list840.size; ++_i842)
{
- _elem833 = iprot.readI64();
- struct.fileIds.add(_elem833);
+ _elem841 = iprot.readI64();
+ struct.fileIds.add(_elem841);
}
iprot.readListEnd();
}
@@ -383,9 +383,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(FILE_IDS_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, struct.fileIds.size()));
- for (long _iter835 : struct.fileIds)
+ for (long _iter843 : struct.fileIds)
{
- oprot.writeI64(_iter835);
+ oprot.writeI64(_iter843);
}
oprot.writeListEnd();
}
@@ -410,9 +410,9 @@ import org.slf4j.LoggerFactory;
TTupleProtocol oprot = (TTupleProtocol) prot;
{
oprot.writeI32(struct.fileIds.size());
- for (long _iter836 : struct.fileIds)
+ for (long _iter844 : struct.fileIds)
{
- oprot.writeI64(_iter836);
+ oprot.writeI64(_iter844);
}
}
}
@@ -421,13 +421,13 @@ import org.slf4j.LoggerFactory;
public void read(org.apache.thrift.protocol.TProtocol prot, ClearFileMetadataRequest struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
{
- org.apache.thrift.protocol.TList _list837 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
- struct.fileIds = new ArrayList<Long>(_list837.size);
- long _elem838;
- for (int _i839 = 0; _i839 < _list837.size; ++_i839)
+ org.apache.thrift.protocol.TList _list845 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64, iprot.readI32());
+ struct.fileIds = new ArrayList<Long>(_list845.size);
+ long _elem846;
+ for (int _i847 = 0; _i847 < _list845.size; ++_i847)
{
- _elem838 = iprot.readI64();
- struct.fileIds.add(_elem838);
+ _elem846 = iprot.readI64();
+ struct.fileIds.add(_elem846);
}
}
struct.setFileIdsIsSet(true);
http://git-wip-us.apache.org/repos/asf/hive/blob/10cfba20/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapabilities.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapabilities.java b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapabilities.java
index f5c9582..17f8b77 100644
--- a/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapabilities.java
+++ b/standalone-metastore/metastore-common/src/gen/thrift/gen-javabean/org/apache/hadoop/hive/metastore/api/ClientCapabilities.java
@@ -354,13 +354,13 @@ import org.slf4j.LoggerFactory;
case 1: // VALUES
if (schemeField.type == org.apache.thrift.protocol.TType.LIST) {
{
- org.apache.thrift.protocol.TList _list848 = iprot.readListBegin();
- struct.values = new ArrayList<ClientCapability>(_list848.size);
- ClientCapability _elem849;
- for (int _i850 = 0; _i850 < _list848.size; ++_i850)
+ org.apache.thrift.protocol.TList _list856 = iprot.readListBegin();
+ struct.values = new ArrayList<ClientCapability>(_list856.size);
+ ClientCapability _elem857;
+ for (int _i858 = 0; _i858 < _list856.size; ++_i858)
{
- _elem849 = org.apache.hadoop.hive.metastore.api.ClientCapability.findByValue(iprot.readI32());
- struct.values.add(_elem849);
+ _elem857 = org.apache.hadoop.hive.metastore.api.ClientCapability.findByValue(iprot.readI32());
+ struct.values.add(_elem857);
}
iprot.readListEnd();
}
@@ -386,9 +386,9 @@ import org.slf4j.LoggerFactory;
oprot.writeFieldBegin(VALUES_FIELD_DESC);
{
oprot.writeListBegin(new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, struct.values.size()));
- for (ClientCapability _iter851 : struct.values)
+ for (ClientCapability _iter859 : struct.values)
{
- oprot.writeI32(_iter851.getValue());
+ oprot.writeI32(_iter859.getValue());
}
oprot.writeListEnd();
}
@@ -413,9 +413,9 @@ import org.slf4j.LoggerFactory;
TTupleProtocol oprot = (TTupleProtocol) prot;
{
oprot.writeI32(struct.values.size());
- for (ClientCapability _iter852 : struct.values)
+ for (ClientCapability _iter860 : struct.values)
{
- oprot.writeI32(_iter852.getValue());
+ oprot.writeI32(_iter860.getValue());
}
}
}
@@ -424,13 +424,13 @@ import org.slf4j.LoggerFactory;
public void read(org.apache.thrift.protocol.TProtocol prot, ClientCapabilities struct) throws org.apache.thrift.TException {
TTupleProtocol iprot = (TTupleProtocol) prot;
{
- org.apache.thrift.protocol.TList _list853 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, iprot.readI32());
- struct.values = new ArrayList<ClientCapability>(_list853.size);
- ClientCapability _elem854;
- for (int _i855 = 0; _i855 < _list853.size; ++_i855)
+ org.apache.thrift.protocol.TList _list861 = new org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32, iprot.readI32());
+ struct.values = new ArrayList<ClientCapability>(_list861.size);
+ ClientCapability _elem862;
+ for (int _i863 = 0; _i863 < _list861.size; ++_i863)
{
- _elem854 = org.apache.hadoop.hive.metastore.api.ClientCapability.findByValue(iprot.readI32());
- struct.values.add(_elem854);
+ _elem862 = org.apache.hadoop.hive.metastore.api.ClientCapability.findByValue(iprot.readI32());
+ struct.values.add(_elem862);
}
}
struct.setValuesIsSet(true);