You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2018/12/12 22:34:02 UTC
hive git commit: HIVE-20998: HiveStrictManagedMigration utility
should update DB/Table location as last migration steps (Jason Dere,
reviewed by Ashutosh Chauhan)
Repository: hive
Updated Branches:
refs/heads/master 8d084d676 -> 881e29124
HIVE-20998: HiveStrictManagedMigration utility should update DB/Table location as last migration steps (Jason Dere, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/881e2912
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/881e2912
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/881e2912
Branch: refs/heads/master
Commit: 881e291246a367f0a413da3bb13be410923a37cf
Parents: 8d084d6
Author: Jason Dere <jd...@hortonworks.com>
Authored: Wed Dec 12 14:33:27 2018 -0800
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Wed Dec 12 14:33:27 2018 -0800
----------------------------------------------------------------------
.../ql/util/HiveStrictManagedMigration.java | 38 +++++++++++++++-----
1 file changed, 30 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/881e2912/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
index 9535bed..80025b7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
@@ -404,11 +404,6 @@ public class HiveStrictManagedMigration {
// Set appropriate owner/perms of the DB dir only, no need to recurse
checkAndSetFileOwnerPermissions(fs, newDefaultDbLocation,
ownerName, groupName, dirPerms, null, runOptions.dryRun, false);
-
- // The table processing needs the db location at the old location, so clone the DB object
- // when updating the location.
- Database modifiedDb = dbObj.deepCopy();
- getHiveUpdater().updateDbLocation(modifiedDb, newDefaultDbLocation);
}
}
@@ -416,6 +411,7 @@ public class HiveStrictManagedMigration {
createExternalDbDir(dbObj);
}
+ boolean errorsInThisDb = false;
List<String> tableNames = hms.getTables(dbName, runOptions.tableRegex);
for (String tableName : tableNames) {
// If we did not change the DB location, there is no need to move the table directories.
@@ -424,6 +420,20 @@ public class HiveStrictManagedMigration {
} catch (Exception err) {
LOG.error("Error processing table " + getQualifiedName(dbObj.getName(), tableName), err);
failuresEncountered = true;
+ errorsInThisDb = true;
+ }
+ }
+
+ // Finally update the DB location. This would prevent subsequent runs of the migration from processing this DB.
+ if (modifyDefaultManagedLocation) {
+ if (errorsInThisDb) {
+ LOG.error("Not updating database location for {} since an error was encountered. The migration must be run again for this database.",
+ dbObj.getName());
+ } else {
+ Path newDefaultDbLocation = wh.getDefaultDatabasePath(dbName);
+ // dbObj after this call would have the new DB location.
+ // Keep that in mind if anything below this requires the old DB path.
+ getHiveUpdater().updateDbLocation(dbObj, newDefaultDbLocation);
}
}
}
@@ -590,6 +600,8 @@ public class HiveStrictManagedMigration {
Path oldTablePath = new Path(tableObj.getSd().getLocation());
LOG.info("Moving location of {} from {} to {}", getQualifiedName(tableObj), oldTablePath, newTablePath);
+
+ // Move table directory.
if (!runOptions.dryRun) {
FileSystem fs = newTablePath.getFileSystem(conf);
if (fs.exists(oldTablePath)) {
@@ -601,9 +613,13 @@ public class HiveStrictManagedMigration {
}
}
}
- if (!runOptions.dryRun) {
- getHiveUpdater().updateTableLocation(tableObj, newTablePath);
- }
+
+ // An error occurring between here and before updating the table's location in the metastore
+ // may potentially cause the data to reside in the new location, while the
+ // table/partitions point to the old paths.
+ // The migration would be _REQUIRED_ to run again (and pass) for the data and table/partition
+ // locations to be in sync.
+
if (isPartitionedTable(tableObj)) {
List<String> partNames = hms.listPartitionNames(dbName, tableName, Short.MAX_VALUE);
// TODO: Fetch partitions in batches?
@@ -622,6 +638,12 @@ public class HiveStrictManagedMigration {
}
}
}
+
+ // Finally update the table location. This would prevent this tool from processing this table again
+ // on subsequent runs of the migration.
+ if (!runOptions.dryRun) {
+ getHiveUpdater().updateTableLocation(tableObj, newTablePath);
+ }
}
static void renameFilesToConformToAcid(Table tableObj, IMetaStoreClient hms, Configuration conf, boolean dryRun)