You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2017/03/20 16:10:11 UTC
hive git commit: HIVE-16024: MSCK Repair Requires nonstrict
hive.mapred.mode (Barna Zsombor Klara, reviewed by Sergio Pena,
Vihang Karajgaonkar)
Repository: hive
Updated Branches:
refs/heads/master 99b503a3a -> 031890ca8
HIVE-16024: MSCK Repair Requires nonstrict hive.mapred.mode (Barna Zsombor Klara, reviewed by Sergio Pena, Vihang Karajgaonkar)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/031890ca
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/031890ca
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/031890ca
Branch: refs/heads/master
Commit: 031890ca8d456c8b88bfa60db50fff0d34f29a79
Parents: 99b503a
Author: Barna Zsombor Klara <zs...@cloudera.com>
Authored: Mon Mar 20 11:09:17 2017 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Mon Mar 20 11:09:17 2017 -0500
----------------------------------------------------------------------
.../hive/ql/metadata/HiveMetaStoreChecker.java | 37 ++++++++++++++------
.../test/queries/clientpositive/msck_repair_0.q | 7 ++++
.../results/clientpositive/msck_repair_0.q.out | 8 +++++
3 files changed, 42 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/031890ca/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
index 6805c17..84c0902 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hive.ql.metadata;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
@@ -35,7 +36,10 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import com.google.common.collect.Sets;
+import org.apache.hadoop.hive.common.StringInternUtils;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.ql.log.PerfLogger;
+import org.apache.hadoop.hive.ql.session.SessionState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.fs.FileStatus;
@@ -49,8 +53,6 @@ import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.ql.metadata.CheckResult.PartitionResult;
-import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
-import org.apache.hadoop.hive.ql.parse.PrunedPartitionList;
import org.apache.thrift.TException;
import com.google.common.util.concurrent.MoreExecutors;
@@ -64,6 +66,7 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class HiveMetaStoreChecker {
public static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreChecker.class);
+ public static final String CLASS_NAME = HiveMetaStoreChecker.class.getName();
private final Hive hive;
private final HiveConf conf;
@@ -208,19 +211,28 @@ public class HiveMetaStoreChecker {
return;
}
- List<Partition> parts = new ArrayList<Partition>();
+ PartitionIterable parts;
boolean findUnknownPartitions = true;
if (table.isPartitioned()) {
if (partitions == null || partitions.isEmpty()) {
- PrunedPartitionList prunedPartList =
- PartitionPruner.prune(table, null, conf, toString(), null);
- // no partitions specified, let's get all
- parts.addAll(prunedPartList.getPartitions());
+ String mode = HiveConf.getVar(conf, ConfVars.HIVEMAPREDMODE, (String) null);
+ if ("strict".equalsIgnoreCase(mode)) {
+ parts = new PartitionIterable(hive, table, null, conf.getIntVar(
+ HiveConf.ConfVars.METASTORE_BATCH_RETRIEVE_MAX));
+ } else {
+ List<Partition> loadedPartitions = new ArrayList<>();
+ PerfLogger perfLogger = SessionState.getPerfLogger();
+ perfLogger.PerfLogBegin(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
+ loadedPartitions.addAll(hive.getAllPartitionsOf(table));
+ perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.PARTITION_RETRIEVING);
+ parts = new PartitionIterable(loadedPartitions);
+ }
} else {
// we're interested in specific partitions,
// don't check for any others
findUnknownPartitions = false;
+ List<Partition> loadedPartitions = new ArrayList<>();
for (Map<String, String> map : partitions) {
Partition part = hive.getPartition(table, map, false);
if (part == null) {
@@ -229,10 +241,13 @@ public class HiveMetaStoreChecker {
pr.setPartitionName(Warehouse.makePartPath(map));
result.getPartitionsNotInMs().add(pr);
} else {
- parts.add(part);
+ loadedPartitions.add(part);
}
}
+ parts = new PartitionIterable(loadedPartitions);
}
+ } else {
+ parts = new PartitionIterable(Collections.<Partition>emptyList());
}
checkTable(table, parts, findUnknownPartitions, result);
@@ -255,7 +270,7 @@ public class HiveMetaStoreChecker {
* @throws HiveException
* Could not create Partition object
*/
- void checkTable(Table table, List<Partition> parts,
+ void checkTable(Table table, PartitionIterable parts,
boolean findUnknownPartitions, CheckResult result) throws IOException,
HiveException {
@@ -284,7 +299,9 @@ public class HiveMetaStoreChecker {
}
for (int i = 0; i < partition.getSpec().size(); i++) {
- partPaths.add(partPath.makeQualified(fs));
+ Path qualifiedPath = partPath.makeQualified(fs);
+ StringInternUtils.internUriStringsInPath(qualifiedPath);
+ partPaths.add(qualifiedPath);
partPath = partPath.getParent();
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/031890ca/ql/src/test/queries/clientpositive/msck_repair_0.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/msck_repair_0.q b/ql/src/test/queries/clientpositive/msck_repair_0.q
index ce8ef42..2254233 100644
--- a/ql/src/test/queries/clientpositive/msck_repair_0.q
+++ b/ql/src/test/queries/clientpositive/msck_repair_0.q
@@ -16,4 +16,11 @@ MSCK REPAIR TABLE default.repairtable;
MSCK TABLE repairtable;
+set hive.mapred.mode=strict;
+
+dfs ${system:test.dfs.mkdir} ${system:test.warehouse.dir}/repairtable/p1=e/p2=f/p3=g;
+dfs -touchz ${system:test.warehouse.dir}/repairtable/p1=e/p2=f/p3=g/datafile;
+
+MSCK REPAIR TABLE default.repairtable;
+
DROP TABLE default.repairtable;
http://git-wip-us.apache.org/repos/asf/hive/blob/031890ca/ql/src/test/results/clientpositive/msck_repair_0.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/msck_repair_0.q.out b/ql/src/test/results/clientpositive/msck_repair_0.q.out
index 3f2fe75..2e0d9dc 100644
--- a/ql/src/test/results/clientpositive/msck_repair_0.q.out
+++ b/ql/src/test/results/clientpositive/msck_repair_0.q.out
@@ -37,6 +37,14 @@ PREHOOK: Output: default@repairtable
POSTHOOK: query: MSCK TABLE repairtable
POSTHOOK: type: MSCK
POSTHOOK: Output: default@repairtable
+PREHOOK: query: MSCK REPAIR TABLE default.repairtable
+PREHOOK: type: MSCK
+PREHOOK: Output: default@repairtable
+POSTHOOK: query: MSCK REPAIR TABLE default.repairtable
+POSTHOOK: type: MSCK
+POSTHOOK: Output: default@repairtable
+Partitions not in metastore: repairtable:p1=e/p2=f
+Repair: Added partition to metastore default.repairtable:p1=e/p2=f
PREHOOK: query: DROP TABLE default.repairtable
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@repairtable