You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sz...@apache.org on 2020/04/08 09:43:10 UTC
[hive] branch master updated: HIVE-23142:
HiveStrictManagedMigration fails with tables that have null location (Adam
Szita, reviewed by Marta Kuczora)
This is an automated email from the ASF dual-hosted git repository.
szita pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d91cc0c HIVE-23142: HiveStrictManagedMigration fails with tables that have null location (Adam Szita, reviewed by Marta Kuczora)
d91cc0c is described below
commit d91cc0cd84b7d0ecc0f29d44b109b46e21194eec
Author: Adam Szita <sz...@cloudera.com>
AuthorDate: Tue Apr 7 14:31:40 2020 +0200
HIVE-23142: HiveStrictManagedMigration fails with tables that have null location (Adam Szita, reviewed by Marta Kuczora)
---
.../hadoop/hive/ql/util/HiveStrictManagedMigration.java | 17 +++++++++++++----
.../hive/ql/util/TestHiveStrictManagedMigration.java | 15 +++++++++++++++
2 files changed, 28 insertions(+), 4 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
index e2e5983..5e658a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/util/HiveStrictManagedMigration.java
@@ -34,6 +34,7 @@ import org.apache.commons.cli.GnuParser;
import org.apache.commons.cli.HelpFormatter;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
+import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -80,6 +81,8 @@ import static org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE;
public class HiveStrictManagedMigration {
private static final Logger LOG = LoggerFactory.getLogger(HiveStrictManagedMigration.class);
+ @VisibleForTesting
+ static int RC = 0;
public enum TableMigrationOption {
NONE, // Do nothing
@@ -194,6 +197,7 @@ public class HiveStrictManagedMigration {
public static void main(String[] args) throws Exception {
RunOptions runOptions;
+ RC = 0;
try {
Options opts = createOptions();
@@ -210,7 +214,6 @@ public class HiveStrictManagedMigration {
throw new Exception("Error processing options", err);
}
- int rc = 0;
HiveStrictManagedMigration migration = null;
try {
HiveConf conf = hiveConf == null ? new HiveConf() : hiveConf;
@@ -227,7 +230,7 @@ public class HiveStrictManagedMigration {
migration.run();
} catch (Exception err) {
LOG.error("Failed with error", err);
- rc = -1;
+ RC = -1;
} finally {
if (migration != null) {
migration.cleanup();
@@ -236,7 +239,7 @@ public class HiveStrictManagedMigration {
// TODO: Something is preventing the process from terminating after main(), adding exit() as hacky solution.
if (hiveConf == null) {
- System.exit(rc);
+ System.exit(RC);
}
}
@@ -747,7 +750,13 @@ public class HiveStrictManagedMigration {
return true;
}
- Path tablePath = new Path(tableObj.getSd().getLocation());
+ String tablePathString = tableObj.getSd().getLocation();
+ if (StringUtils.isEmpty(tablePathString)) {
+ // When using this tool in full automatic mode (no DB/table regexes and automatic migration option) we may
+ // encounter sysdb / information_schema databases. These should not be moved, they have null location.
+ return true;
+ }
+ Path tablePath = new Path(tablePathString);
boolean shouldMoveTable = modifyLocation && (
(MANAGED_TABLE.name().equals(tableObj.getTableType()) && runOptions.shouldModifyManagedTableLocation) ||
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java b/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java
index 16d4772..038a8a2 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/util/TestHiveStrictManagedMigration.java
@@ -25,6 +25,7 @@ import static org.apache.hadoop.hive.ql.TxnCommandsBaseForTests.Table.NONACIDORC
import static org.apache.hadoop.hive.ql.TxnCommandsBaseForTests.Table.NONACIDORCTBL2;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
@@ -34,13 +35,17 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
+import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
+import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.ql.TxnCommandsBaseForTests;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
+import com.google.common.collect.ImmutableMap;
+
import org.junit.Assert;
import org.junit.Test;
@@ -65,6 +70,13 @@ public class TestHiveStrictManagedMigration extends TxnCommandsBaseForTests {
runStatementOnDriver(
"CREATE EXTERNAL TABLE texternal (a int, b int)");
+ // Case for table having null location
+ runStatementOnDriver("CREATE EXTERNAL TABLE test.sysdbtest(tbl_id bigint)");
+ org.apache.hadoop.hive.ql.metadata.Table table = Hive.get(hiveConf).getTable("test", "sysdbtest");
+ table.getSd().unsetLocation();
+ Hive.get(hiveConf).alterTable(table, false,
+ new EnvironmentContext(ImmutableMap.of(StatsSetupConst.DO_NOT_UPDATE_STATS, StatsSetupConst.TRUE)), false);
+
String oldWarehouse = getWarehouseDir();
String[] args = {"--hiveconf", "hive.strict.managed.tables=true", "-m", "automatic", "--modifyManagedTables",
"--oldWarehouseRoot", oldWarehouse};
@@ -138,6 +150,9 @@ public class TestHiveStrictManagedMigration extends TxnCommandsBaseForTests {
HiveStrictManagedMigration.hiveConf = hiveConf;
HiveStrictManagedMigration.scheme = "file";
HiveStrictManagedMigration.main(args);
+ if (HiveStrictManagedMigration.RC != 0) {
+ fail("HiveStrictManagedMigration failed with error(s)");
+ }
}
private void setupExternalTableTest() throws Exception {