You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2017/03/01 22:10:54 UTC
hive git commit: HIVE-16014: HiveMetastoreChecker should use
hive.metastore.fshandler.threads instead of hive.mv.files.thread for pool
size (Vihang Karajgaonkar, reviewed by Sahil Takiar, Sergio Pena,
Rajesh Balamohan)
Repository: hive
Updated Branches:
refs/heads/master b9ad6dc38 -> 92aaed243
HIVE-16014: HiveMetastoreChecker should use hive.metastore.fshandler.threads instead of hive.mv.files.thread for pool size (Vihang Karajgaonkar, reviewed by Sahil Takiar, Sergio Pena, Rajesh Balamohan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/92aaed24
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/92aaed24
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/92aaed24
Branch: refs/heads/master
Commit: 92aaed243c27c73f9168c99c2837431477c4e41c
Parents: b9ad6dc
Author: Vihang Karajgaonkar <vi...@cloudera.com>
Authored: Wed Mar 1 16:09:50 2017 -0600
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed Mar 1 16:09:50 2017 -0600
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/92aaed24/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
index 5d2ae2c..0859022 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java
@@ -404,7 +404,12 @@ public class HiveMetaStoreChecker {
*/
private void checkPartitionDirs(Path basePath, Set<Path> allDirs, int maxDepth) throws IOException, HiveException {
- int poolSize = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 15);
+ // Here we just reuse the THREAD_COUNT configuration for
+ // METASTORE_FS_HANDLER_THREADS_COUNT since this results in better performance
+ // The number of missing partitions discovered are later added by metastore using a
+ // threadpool of size METASTORE_FS_HANDLER_THREADS_COUNT. If we have different sized
+ // pool here the smaller sized pool of the two becomes a bottleneck
+ int poolSize = conf.getInt(ConfVars.METASTORE_FS_HANDLER_THREADS_COUNT.varname, 15);
// Check if too low config is provided for move files. 2x CPU is reasonable max count.
poolSize = poolSize == 0 ? poolSize : Math.max(poolSize,