You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sa...@apache.org on 2020/05/29 06:54:26 UTC
[hive] branch master updated: HIVE-23347: MSCK REPAIR cannot
discover partitions with upper case directory names(Adesh Kumar Rao,
reviewed by Sankar Hariappan)
This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 6c8d478 HIVE-23347: MSCK REPAIR cannot discover partitions with upper case directory names(Adesh Kumar Rao, reviewed by Sankar Hariappan)
6c8d478 is described below
commit 6c8d478af1a7c2ea8478535a938a0585d000ec8f
Author: Adesh Kumar Rao <ad...@gmail.com>
AuthorDate: Fri May 29 12:22:52 2020 +0530
HIVE-23347: MSCK REPAIR cannot discover partitions with upper case directory names(Adesh Kumar Rao, reviewed by Sankar Hariappan)
Signed-off-by: Sankar Hariappan <sa...@apache.org>
---
.../test/resources/testconfiguration.properties | 3 ++
.../ql/exec/TestMsckCreatePartitionsInBatches.java | 1 +
ql/src/test/queries/clientnegative/msck_repair_5.q | 16 +++++++
ql/src/test/queries/clientnegative/msck_repair_6.q | 15 +++++++
ql/src/test/queries/clientpositive/msck_repair_4.q | 14 +++++++
ql/src/test/queries/clientpositive/msck_repair_5.q | 15 +++++++
ql/src/test/queries/clientpositive/msck_repair_6.q | 13 ++++++
.../results/clientnegative/msck_repair_5.q.out | 30 +++++++++++++
.../results/clientnegative/msck_repair_6.q.out | 30 +++++++++++++
.../results/clientpositive/msck_repair_4.q.out | 48 +++++++++++++++++++++
.../results/clientpositive/msck_repair_5.q.out | 49 ++++++++++++++++++++++
.../results/clientpositive/msck_repair_6.q.out | 49 ++++++++++++++++++++++
.../apache/hadoop/hive/metastore/CheckResult.java | 41 ++++++++++++++++++
.../hive/metastore/HiveMetaStoreChecker.java | 29 ++++++++++---
.../org/apache/hadoop/hive/metastore/Msck.java | 2 +-
.../hive/metastore/utils/MetaStoreServerUtils.java | 10 +++--
16 files changed, 356 insertions(+), 9 deletions(-)
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 1fd09eb..f430a13 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -226,6 +226,9 @@ mr.query.files=\
msck_repair_1.q,\
msck_repair_2.q,\
msck_repair_3.q,\
+ msck_repair_4.q,\
+ msck_repair_5.q,\
+ msck_repair_6.q,\
msck_repair_acid.q,\
msck_repair_batchsize.q,\
msck_repair_drop.q,\
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java
index 869249a..30c8932 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/TestMsckCreatePartitionsInBatches.java
@@ -137,6 +137,7 @@ public class TestMsckCreatePartitionsInBatches {
for (int i = 0; i < numOfParts; i++) {
PartitionResult result = new PartitionResult();
result.setPartitionName("city=dummyCity_" + String.valueOf(i));
+ result.setTableName("dummyTable");
partsNotInMs.add(result);
}
return partsNotInMs;
diff --git a/ql/src/test/queries/clientnegative/msck_repair_5.q b/ql/src/test/queries/clientnegative/msck_repair_5.q
new file mode 100644
index 0000000..fe1eab7
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/msck_repair_5.q
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS repairtable;
+
+CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING);
+
+MSCK REPAIR TABLE default.repairtable;
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=A;
+
+MSCK REPAIR TABLE default.repairtable;
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=a;
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/P1=a;
+
+MSCK REPAIR TABLE default.repairtable;
+
+DROP TABLE default.repairtable;
diff --git a/ql/src/test/queries/clientnegative/msck_repair_6.q b/ql/src/test/queries/clientnegative/msck_repair_6.q
new file mode 100644
index 0000000..76e8557
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/msck_repair_6.q
@@ -0,0 +1,15 @@
+DROP TABLE IF EXISTS repairtable;
+
+CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING);
+
+MSCK REPAIR TABLE default.repairtable;
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=a;
+
+MSCK REPAIR TABLE default.repairtable;
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/P1=a;
+
+MSCK REPAIR TABLE default.repairtable;
+
+DROP TABLE default.repairtable;
\ No newline at end of file
diff --git a/ql/src/test/queries/clientpositive/msck_repair_4.q b/ql/src/test/queries/clientpositive/msck_repair_4.q
new file mode 100644
index 0000000..68ddaea
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/msck_repair_4.q
@@ -0,0 +1,14 @@
+DROP TABLE IF EXISTS repairtable_n4;
+
+CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC;
+
+MSCK REPAIR TABLE repairtable_n4;
+show partitions repairtable_n4;
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/Year=2020/Month=3/Day=1;
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n4/Year=2020/Month=3/Day=2;
+
+MSCK REPAIR TABLE repairtable_n4;
+show partitions repairtable_n4;
+
+DROP TABLE default.repairtable_n4;
diff --git a/ql/src/test/queries/clientpositive/msck_repair_5.q b/ql/src/test/queries/clientpositive/msck_repair_5.q
new file mode 100644
index 0000000..677063b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/msck_repair_5.q
@@ -0,0 +1,15 @@
+DROP TABLE IF EXISTS repairtable_n5;
+
+CREATE EXTERNAL TABLE repairtable_n5(key INT, value STRING) PARTITIONED BY (Country String) stored as ORC;
+
+MSCK REPAIR TABLE repairtable_n5;
+show partitions repairtable_n5;
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/Country=US;
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/Country=us;
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable_n5/Country=India;
+
+MSCK REPAIR TABLE repairtable_n5;
+show partitions repairtable_n5;
+
+DROP TABLE default.repairtable_n5;
diff --git a/ql/src/test/queries/clientpositive/msck_repair_6.q b/ql/src/test/queries/clientpositive/msck_repair_6.q
new file mode 100644
index 0000000..b32248d
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/msck_repair_6.q
@@ -0,0 +1,13 @@
+DROP TABLE IF EXISTS repairtable_n6;
+
+CREATE EXTERNAL TABLE repairtable_n6(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC location '${system:test.tmp.dir}/apps/hive/warehouse/test.db/Repairtable_n6/';
+
+MSCK REPAIR TABLE repairtable_n6;
+show partitions repairtable_n6;
+
+dfs ${system:test.dfs.mkdir} -p ${system:test.tmp.dir}/apps/hive/warehouse/test.db/Repairtable_n6/Year=2020/Month=4/Day=1;
+
+MSCK REPAIR TABLE repairtable_n6;
+show partitions repairtable_n6;
+
+DROP TABLE default.repairtable_n6;
diff --git a/ql/src/test/results/clientnegative/msck_repair_5.q.out b/ql/src/test/results/clientnegative/msck_repair_5.q.out
new file mode 100644
index 0000000..b06dde7
--- /dev/null
+++ b/ql/src/test/results/clientnegative/msck_repair_5.q.out
@@ -0,0 +1,30 @@
+PREHOOK: query: DROP TABLE IF EXISTS repairtable
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable
+POSTHOOK: query: CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable
+PREHOOK: query: MSCK REPAIR TABLE default.repairtable
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable
+POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable
+PREHOOK: query: MSCK REPAIR TABLE default.repairtable
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable
+POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable
+Partitions not in metastore: repairtable:p1=A
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE default.repairtable
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask
diff --git a/ql/src/test/results/clientnegative/msck_repair_6.q.out b/ql/src/test/results/clientnegative/msck_repair_6.q.out
new file mode 100644
index 0000000..680b009
--- /dev/null
+++ b/ql/src/test/results/clientnegative/msck_repair_6.q.out
@@ -0,0 +1,30 @@
+PREHOOK: query: DROP TABLE IF EXISTS repairtable
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable
+POSTHOOK: query: CREATE TABLE repairtable(col STRING) PARTITIONED BY (p1 STRING)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable
+PREHOOK: query: MSCK REPAIR TABLE default.repairtable
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable
+POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable
+PREHOOK: query: MSCK REPAIR TABLE default.repairtable
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable
+POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable
+Partitions not in metastore: repairtable:p1=a
+#### A masked pattern was here ####
+PREHOOK: query: MSCK REPAIR TABLE default.repairtable
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.ddl.DDLTask
diff --git a/ql/src/test/results/clientpositive/msck_repair_4.q.out b/ql/src/test/results/clientpositive/msck_repair_4.q.out
new file mode 100644
index 0000000..aca6654
--- /dev/null
+++ b/ql/src/test/results/clientpositive/msck_repair_4.q.out
@@ -0,0 +1,48 @@
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n4
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n4
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n4
+POSTHOOK: query: CREATE EXTERNAL TABLE repairtable_n4(key INT, value STRING) PARTITIONED BY (Year INT, Month INT, Day INT) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n4
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n4
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n4
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n4
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n4
+PREHOOK: query: show partitions repairtable_n4
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n4
+POSTHOOK: query: show partitions repairtable_n4
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n4
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n4
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n4
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n4
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n4
+Partitions not in metastore: repairtable_n4:year=2020/month=3/day=1 repairtable_n4:year=2020/month=3/day=2
+#### A masked pattern was here ####
+PREHOOK: query: show partitions repairtable_n4
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n4
+POSTHOOK: query: show partitions repairtable_n4
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n4
+year=2020/month=3/day=1
+year=2020/month=3/day=2
+PREHOOK: query: DROP TABLE default.repairtable_n4
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n4
+PREHOOK: Output: default@repairtable_n4
+POSTHOOK: query: DROP TABLE default.repairtable_n4
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n4
+POSTHOOK: Output: default@repairtable_n4
diff --git a/ql/src/test/results/clientpositive/msck_repair_5.q.out b/ql/src/test/results/clientpositive/msck_repair_5.q.out
new file mode 100644
index 0000000..4f07ecb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/msck_repair_5.q.out
@@ -0,0 +1,49 @@
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n5
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n5
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE EXTERNAL TABLE repairtable_n5(key INT, value STRING) PARTITIONED BY (Country String) stored as ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n5
+POSTHOOK: query: CREATE EXTERNAL TABLE repairtable_n5(key INT, value STRING) PARTITIONED BY (Country String) stored as ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n5
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n5
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n5
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n5
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n5
+PREHOOK: query: show partitions repairtable_n5
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n5
+POSTHOOK: query: show partitions repairtable_n5
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n5
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n5
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n5
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n5
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n5
+Partitions not in metastore: repairtable_n5:country=India repairtable_n5:country=US repairtable_n5:country=us
+#### A masked pattern was here ####
+PREHOOK: query: show partitions repairtable_n5
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n5
+POSTHOOK: query: show partitions repairtable_n5
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n5
+country=India
+country=US
+country=us
+PREHOOK: query: DROP TABLE default.repairtable_n5
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n5
+PREHOOK: Output: default@repairtable_n5
+POSTHOOK: query: DROP TABLE default.repairtable_n5
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n5
+POSTHOOK: Output: default@repairtable_n5
diff --git a/ql/src/test/results/clientpositive/msck_repair_6.q.out b/ql/src/test/results/clientpositive/msck_repair_6.q.out
new file mode 100644
index 0000000..92986e4
--- /dev/null
+++ b/ql/src/test/results/clientpositive/msck_repair_6.q.out
@@ -0,0 +1,49 @@
+PREHOOK: query: DROP TABLE IF EXISTS repairtable_n6
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS repairtable_n6
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@repairtable_n6
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@repairtable_n6
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n6
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n6
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n6
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n6
+PREHOOK: query: show partitions repairtable_n6
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n6
+POSTHOOK: query: show partitions repairtable_n6
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n6
+PREHOOK: query: MSCK REPAIR TABLE repairtable_n6
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable_n6
+POSTHOOK: query: MSCK REPAIR TABLE repairtable_n6
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable_n6
+Partitions not in metastore: repairtable_n6:year=2020/month=4/day=1
+#### A masked pattern was here ####
+PREHOOK: query: show partitions repairtable_n6
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@repairtable_n6
+POSTHOOK: query: show partitions repairtable_n6
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@repairtable_n6
+year=2020/month=4/day=1
+PREHOOK: query: DROP TABLE default.repairtable_n6
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@repairtable_n6
+PREHOOK: Output: default@repairtable_n6
+POSTHOOK: query: DROP TABLE default.repairtable_n6
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@repairtable_n6
+POSTHOOK: Output: default@repairtable_n6
diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java
index 5287f47..07af0ee 100644
--- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java
+++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/CheckResult.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.hive.metastore;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+
+import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
@@ -31,6 +35,7 @@ public class CheckResult {
private Set<PartitionResult> partitionsNotOnFs = new TreeSet<PartitionResult>();
private Set<PartitionResult> partitionsNotInMs = new TreeSet<PartitionResult>();
private Set<PartitionResult> expiredPartitions = new TreeSet<>();
+ private Set<PartitionResult> correctPartitions = new TreeSet<>();
/**
* @return a list of tables not found on the filesystem.
@@ -101,6 +106,14 @@ public class CheckResult {
this.expiredPartitions = expiredPartitions;
}
+ public Set<PartitionResult> getCorrectPartitions() {
+ return this.correctPartitions;
+ }
+
+ public void setCorrectPartitions(final Set<PartitionResult> correctPartitions) {
+ this.correctPartitions = correctPartitions;
+ }
+
/**
* A basic description of a partition that is missing from either the fs or
* the ms.
@@ -108,6 +121,7 @@ public class CheckResult {
public static class PartitionResult implements Comparable<PartitionResult> {
private String partitionName;
private String tableName;
+ private Path path;
/**
* @return name of partition
@@ -139,11 +153,38 @@ public class CheckResult {
this.tableName = tableName;
}
+ public void setPath(Path path) {
+ this.path = path;
+ }
+
+ public Path getLocation(Path tablePath, Map<String, String> partSpec) throws MetaException {
+ if (this.path == null) {
+ return new Path(tablePath, Warehouse.makePartPath(partSpec));
+ }
+
+ return this.path;
+ }
+
@Override
public String toString() {
return tableName + ":" + partitionName;
}
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof PartitionResult) {
+ if (0 == compareTo((PartitionResult)other)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return super.hashCode();
+ }
+
public int compareTo(PartitionResult o) {
int ret = tableName.compareTo(o.tableName);
return ret != 0 ? ret : partitionName.compareTo(o.partitionName);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
index 9b454cc..c15b341 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java
@@ -320,11 +320,14 @@ public class HiveMetaStoreChecker {
continue;
}
fs = partPath.getFileSystem(conf);
+
+ CheckResult.PartitionResult prFromMetastore = new CheckResult.PartitionResult();
+ prFromMetastore.setPartitionName(getPartitionName(table, partition));
+ prFromMetastore.setTableName(partition.getTableName());
if (!fs.exists(partPath)) {
- CheckResult.PartitionResult pr = new CheckResult.PartitionResult();
- pr.setPartitionName(getPartitionName(table, partition));
- pr.setTableName(partition.getTableName());
- result.getPartitionsNotOnFs().add(pr);
+ result.getPartitionsNotOnFs().add(prFromMetastore);
+ } else {
+ result.getCorrectPartitions().add(prFromMetastore);
}
if (partitionExpirySeconds > 0) {
@@ -403,8 +406,24 @@ public class HiveMetaStoreChecker {
CheckResult.PartitionResult pr = new CheckResult.PartitionResult();
pr.setPartitionName(partitionName);
pr.setTableName(table.getTableName());
-
+ // Also set the correct partition path here as creating path from Warehouse.makePartPath will always return
+ // lowercase keys/path. Even if we add the new partition with lowerkeys, get queries on such partition
+ // will not return any results.
+ pr.setPath(partPath);
+
+ // Check if partition already exists. No need to check for those partition which are present in db
+ // but no in fs as msck will override the partition location in db
+ if (result.getCorrectPartitions().contains(pr)) {
+ String msg = "The partition '" + pr.toString() + "' already exists for table" + table.getTableName();
+ throw new MetastoreException(msg);
+ } else if (result.getPartitionsNotInMs().contains(pr)) {
+ String msg = "Found two paths for same partition '" + pr.toString() + "' for table " + table.getTableName();
+ throw new MetastoreException(msg);
+ }
result.getPartitionsNotInMs().add(pr);
+ if (result.getPartitionsNotOnFs().contains(pr)) {
+ result.getPartitionsNotOnFs().remove(pr);
+ }
}
}
LOG.debug("Number of partitions not in metastore : " + result.getPartitionsNotInMs().size());
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java
index 771874b..61ac997 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/Msck.java
@@ -377,7 +377,7 @@ public class Msck {
continue;
}
Map<String, String> partSpec = Warehouse.makeSpecFromName(part.getPartitionName());
- Path location = new Path(tablePath, Warehouse.makePartPath(partSpec));
+ Path location = part.getLocation(tablePath, partSpec);
Partition partition = MetaStoreServerUtils.createMetaPartitionObject(table, partSpec, location);
partition.setWriteId(table.getWriteId());
partsToAdd.add(partition);
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
index 92d10cd..f9d7c13 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/utils/MetaStoreServerUtils.java
@@ -1449,12 +1449,16 @@ public class MetaStoreServerUtils {
return result;
}
- String partitionName = parts[0];
+ // Since hive stores partitions keys in lower case, if the hdfs path contains mixed case,
+ // it should be converted to lower case
+ String partitionName = parts[0].toLowerCase();
+ // Do not convert the partitionValue to lowercase
+ String partitionValue = parts[1];
if (partCols.contains(partitionName)) {
if (result == null) {
- result = currPath.getName();
+ result = partitionName + "=" + partitionValue;
} else {
- result = currPath.getName() + Path.SEPARATOR + result;
+ result = partitionName + "=" + partitionValue + Path.SEPARATOR + result;
}
}
}