You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2020/05/27 18:15:33 UTC
[hive] branch master updated: HIVE-23488 : Optimise
PartitionManagementTask::Msck::repair (Rajesh Balamohan via Ashutosh
Chauhan)
This is an automated email from the ASF dual-hosted git repository.
hashutosh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new a3a25eb HIVE-23488 : Optimise PartitionManagementTask::Msck::repair (Rajesh Balamohan via Ashutosh Chauhan)
a3a25eb is described below
commit a3a25ebb7e65d3348213a2e79f629eaffd9605c6
Author: Rajesh Balamohan <rb...@apache.org>
AuthorDate: Wed May 27 11:14:52 2020 -0700
HIVE-23488 : Optimise PartitionManagementTask::Msck::repair (Rajesh Balamohan via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
---
.../hive/ql/metadata/TestHiveMetaStoreChecker.java | 48 +++++++++++-----------
.../hive/metastore/HiveMetaStoreChecker.java | 26 ++++++------
.../org/apache/hadoop/hive/metastore/Msck.java | 2 +-
3 files changed, 39 insertions(+), 37 deletions(-)
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
index 520eb1b..198fb81 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/metadata/TestHiveMetaStoreChecker.java
@@ -116,7 +116,7 @@ public class TestHiveMetaStoreChecker {
@Test
public void testTableCheck() throws HiveException, IOException, TException, MetastoreException {
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, null, null, result);
+ checker.checkMetastore(catName, dbName, null, null, null, result);
// we haven't added anything so should return an all ok
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
@@ -125,7 +125,7 @@ public class TestHiveMetaStoreChecker {
// check table only, should not exist in ms
result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(1, result.getTablesNotInMs().size());
assertEquals(tableName, result.getTablesNotInMs().iterator().next());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
@@ -148,7 +148,7 @@ public class TestHiveMetaStoreChecker {
// now we've got a table, check that it works
// first check all (1) tables
result = new CheckResult();
- checker.checkMetastore(catName, dbName, null, null, result);
+ checker.checkMetastore(catName, dbName, null, null, null, result);
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
@@ -156,7 +156,7 @@ public class TestHiveMetaStoreChecker {
// then let's check the one we know about
result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
@@ -168,7 +168,7 @@ public class TestHiveMetaStoreChecker {
// now this shouldn't find the path on the fs
result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(1, result.getTablesNotOnFs().size());
assertEquals(tableName, result.getTablesNotOnFs().iterator().next());
@@ -184,7 +184,7 @@ public class TestHiveMetaStoreChecker {
// find the extra table
result = new CheckResult();
- checker.checkMetastore(catName, dbName, null, null, result);
+ checker.checkMetastore(catName, dbName, null, null, null, result);
assertEquals(1, result.getTablesNotInMs().size());
assertEquals(fakeTable.getName(), Lists.newArrayList(result.getTablesNotInMs()).get(0));
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
@@ -198,7 +198,7 @@ public class TestHiveMetaStoreChecker {
// should return all ok
result = new CheckResult();
- checker.checkMetastore(catName, dbName, null, null, result);
+ checker.checkMetastore(catName, dbName, null, null, null, result);
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
@@ -222,7 +222,7 @@ public class TestHiveMetaStoreChecker {
fs.mkdirs(fakePart);
fs.deleteOnExit(fakePart);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs());
@@ -242,7 +242,7 @@ public class TestHiveMetaStoreChecker {
"fakedate=2009-01-01/fakecity=sanjose");
fs.mkdirs(fakePart);
fs.deleteOnExit(fakePart);
- checker.checkMetastore(catName, dbName, tableName, null, new CheckResult());
+ checker.checkMetastore(catName, dbName, tableName, null, null, new CheckResult());
}
/*
@@ -265,7 +265,7 @@ public class TestHiveMetaStoreChecker {
fs.deleteOnExit(fakePart);
createPartitionsDirectoriesOnFS(table, 2);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs());
@@ -301,7 +301,7 @@ public class TestHiveMetaStoreChecker {
Table table = createTestTable();
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
// all is well
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
@@ -320,7 +320,7 @@ public class TestHiveMetaStoreChecker {
fs.delete(partToRemovePath, true);
result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
// missing one partition on fs
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
@@ -335,7 +335,7 @@ public class TestHiveMetaStoreChecker {
partsCopy.add(partitions.get(1).getSpec());
// check only the partition that exists, all should be well
result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, partsCopy, result);
+ checker.checkMetastore(catName, dbName, tableName, partsCopy, null, result);
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
@@ -347,7 +347,7 @@ public class TestHiveMetaStoreChecker {
hive.dropTable(dbName, tableName, true, true);
hive.createTable(table);
result = new CheckResult();
- checker.checkMetastore(catName, dbName, null, null, result);
+ checker.checkMetastore(catName, dbName, null, null, null, result);
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
@@ -398,7 +398,7 @@ public class TestHiveMetaStoreChecker {
// add 10 partitions on the filesystem
createPartitionsDirectoriesOnFS(testTable, 10);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String>emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String>emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult>emptySet(), result.getPartitionsNotOnFs());
@@ -417,7 +417,7 @@ public class TestHiveMetaStoreChecker {
// add 10 partitions on the filesystem
createPartitionsDirectoriesOnFS(testTable, 10);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs());
@@ -442,7 +442,7 @@ public class TestHiveMetaStoreChecker {
// add 10 partitions on the filesystem
createPartitionsDirectoriesOnFS(testTable, 10);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs());
@@ -466,7 +466,7 @@ public class TestHiveMetaStoreChecker {
// add 10 partitions on the filesystem
createPartitionsDirectoriesOnFS(testTable, 10);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs());
@@ -494,7 +494,7 @@ public class TestHiveMetaStoreChecker {
CheckResult result = new CheckResult();
Exception exception = null;
try {
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
} catch (Exception e) {
exception = e;
}
@@ -503,7 +503,7 @@ public class TestHiveMetaStoreChecker {
result = new CheckResult();
exception = null;
try {
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
} catch (Exception e) {
exception = e;
}
@@ -525,7 +525,7 @@ public class TestHiveMetaStoreChecker {
// add 10 partitions on the filesystem
createInvalidPartitionDirsOnFS(testTable, 10);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
}
/*
@@ -543,7 +543,7 @@ public class TestHiveMetaStoreChecker {
// add 10 partitions on the filesystem
createPartitionsDirectoriesOnFS(testTable, 2);
CheckResult result = new CheckResult();
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs());
assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs());
assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs());
@@ -572,7 +572,7 @@ public class TestHiveMetaStoreChecker {
CheckResult result = new CheckResult();
Exception exception = null;
try {
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
} catch (Exception e) {
exception = e;
}
@@ -581,7 +581,7 @@ public class TestHiveMetaStoreChecker {
result = new CheckResult();
exception = null;
try {
- checker.checkMetastore(catName, dbName, tableName, null, result);
+ checker.checkMetastore(catName, dbName, tableName, null, null, result);
} catch (Exception e) {
exception = e;
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
index 6f4400a..9b454cc 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
@@ -111,6 +111,7 @@ public class HiveMetaStoreChecker {
* @param partitions
* List of partition name value pairs, if null or empty check all
* partitions
+ * @param table
* @param result
* Fill this with the results of the check
* @throws MetastoreException
@@ -119,7 +120,7 @@ public class HiveMetaStoreChecker {
* Most likely filesystem related
*/
public void checkMetastore(String catName, String dbName, String tableName,
- List<? extends Map<String, String>> partitions, CheckResult result)
+ List<? extends Map<String, String>> partitions, Table table, CheckResult result)
throws MetastoreException, IOException {
if (dbName == null || "".equalsIgnoreCase(dbName)) {
@@ -131,16 +132,16 @@ public class HiveMetaStoreChecker {
// no table specified, check all tables and all partitions.
List<String> tables = getMsc().getTables(catName, dbName, ".*");
for (String currentTableName : tables) {
- checkTable(catName, dbName, currentTableName, null, result);
+ checkTable(catName, dbName, currentTableName, null, null, result);
}
findUnknownTables(catName, dbName, tables, result);
} else if (partitions == null || partitions.isEmpty()) {
// only one table, let's check all partitions
- checkTable(catName, dbName, tableName, null, result);
+ checkTable(catName, dbName, tableName, null, table, result);
} else {
// check the specified partitions
- checkTable(catName, dbName, tableName, partitions, result);
+ checkTable(catName, dbName, tableName, partitions, table, result);
}
LOG.info("Number of partitionsNotInMs=" + result.getPartitionsNotInMs()
+ ", partitionsNotOnFs=" + result.getPartitionsNotOnFs()
@@ -215,6 +216,7 @@ public class HiveMetaStoreChecker {
* Name of the table
* @param partitions
* Partitions to check, if null or empty get all the partitions.
+ * @param table
* @param result
* Result object
* @throws MetastoreException
@@ -225,16 +227,16 @@ public class HiveMetaStoreChecker {
* Failed to get required information from the metastore.
*/
void checkTable(String catName, String dbName, String tableName,
- List<? extends Map<String, String>> partitions, CheckResult result)
+ List<? extends Map<String, String>> partitions, Table table, CheckResult result)
throws MetaException, IOException, MetastoreException {
- Table table;
-
- try {
- table = getMsc().getTable(catName, dbName, tableName);
- } catch (TException e) {
- result.getTablesNotInMs().add(tableName);
- return;
+ if (table == null) {
+ try {
+ table = getMsc().getTable(catName, dbName, tableName);
+ } catch (TException e) {
+ result.getTablesNotInMs().add(tableName);
+ return;
+ }
}
PartitionIterable parts;
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java
index 59a96e8..771874b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java
@@ -122,7 +122,7 @@ public class Msck {
// And partitions that are not present in filesystem and metadata exists in metastore -
// accessed through getPartitionNotOnFS
checker.checkMetastore(msckInfo.getCatalogName(), msckInfo.getDbName(), msckInfo.getTableName(),
- msckInfo.getPartSpecs(), result);
+ msckInfo.getPartSpecs(), table, result);
Set<CheckResult.PartitionResult> partsNotInMs = result.getPartitionsNotInMs();
Set<CheckResult.PartitionResult> partsNotInFs = result.getPartitionsNotOnFs();
Set<CheckResult.PartitionResult> expiredPartitions = result.getExpiredPartitions();