You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ch...@apache.org on 2023/12/06 04:28:18 UTC
(hive) branch master updated: HIVE-27894: Enhance HMS Handler Logs for all 'get_partition' functions (Shivangi Jha, Reviewed by Chinna Rao Lalam)
This is an automated email from the ASF dual-hosted git repository.
chinnaraol pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1e4f488394d HIVE-27894: Enhance HMS Handler Logs for all 'get_partition' functions (Shivangi Jha, Reviewed by Chinna Rao Lalam)
1e4f488394d is described below
commit 1e4f488394d19ea51766e0633a605e078d8558c3
Author: Shivangi Jha <sh...@gmail.com>
AuthorDate: Wed Dec 6 09:58:12 2023 +0530
HIVE-27894: Enhance HMS Handler Logs for all 'get_partition' functions (Shivangi Jha, Reviewed by Chinna Rao Lalam)
---
.../apache/hadoop/hive/metastore/HMSHandler.java | 92 ++++++++++++++++++----
1 file changed, 75 insertions(+), 17 deletions(-)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index dfabec77ac5..1399553e40b 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -151,6 +151,12 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
static final String NO_FILTER_STRING = "";
static final int UNLIMITED_MAX_PARTITIONS = -1;
+ static final int LOG_SAMPLE_PARTITIONS_MAX_SIZE = 4;
+
+ static final int LOG_SAMPLE_PARTITIONS_HALF_SIZE = 2;
+
+ static final String LOG_SAMPLE_PARTITIONS_SEPARATOR = ",";
+
private Warehouse wh; // hdfs warehouse
private static Striped<Lock> tablelocks;
@@ -815,7 +821,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
private void startPartitionFunction(String function, String cat, String db, String tbl,
List<String> partVals) {
startFunction(function, " : tbl=" +
- TableName.getQualified(cat, db, tbl) + "[" + join(partVals, ",") + "]");
+ TableName.getQualified(cat, db, tbl) + samplePartitionValues(partVals));
}
private void startPartitionFunction(String function, String catName, String db, String tbl,
@@ -824,6 +830,52 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
TableName.getQualified(catName, db, tbl) + " partition=" + partName);
}
+ private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts) {
+ startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Max partitions =" + maxParts);
+ }
+
+ private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
+ List<String> partVals) {
+ startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Max partitions =" + maxParts
+ + samplePartitionValues(partVals));
+ }
+
+ private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
+ String filter) {
+ startFunction(function,
+ " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Filter=" + filter + ": Max partitions ="
+ + maxParts);
+ }
+
+ private void startPartitionFunction(String function, String catName, String db, String tbl, int maxParts,
+ String expression, String defaultPartitionName) {
+ startFunction(function, " : tbl=" + TableName.getQualified(catName, db, tbl) + ": Expression=" + expression
+ + ": Default partition name=" + defaultPartitionName + ": Max partitions=" + maxParts);
+ }
+
+ private String getGroupsCountAndUsername(final String user_name, final List<String> group_names) {
+ return ". Number of groups= " + (group_names == null ? 0 : group_names.size()) + ", user name= " + user_name;
+ }
+
+ private String samplePartitionValues(List<String> partVals) {
+ if (CollectionUtils.isEmpty(partVals)) {
+ return ": Partitions = []";
+ }
+ StringBuilder sb = new StringBuilder(": Number of Partitions = " + partVals.size());
+ sb.append(": Partitions = [");
+ if (partVals.size() > LOG_SAMPLE_PARTITIONS_MAX_SIZE) {
+ // extracting starting 2 values, and ending 2 values
+ sb.append(join(partVals.subList(0, LOG_SAMPLE_PARTITIONS_HALF_SIZE), LOG_SAMPLE_PARTITIONS_SEPARATOR));
+ sb.append(" .... ");
+ sb.append(join(partVals.subList(partVals.size() - LOG_SAMPLE_PARTITIONS_HALF_SIZE, partVals.size()),
+ LOG_SAMPLE_PARTITIONS_SEPARATOR));
+ } else {
+ sb.append(join(partVals, LOG_SAMPLE_PARTITIONS_SEPARATOR));
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
private void endFunction(String function, boolean successful, Exception e) {
endFunction(function, successful, e, null);
}
@@ -5453,8 +5505,9 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
final String user_name, final List<String> group_names)
throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
- startPartitionFunction("get_partition_with_auth", parsedDbName[CAT_NAME],
- parsedDbName[DB_NAME], tbl_name, part_vals);
+ startFunction("get_partition_with_auth",
+ " : tbl=" + TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name)
+ + samplePartitionValues(part_vals) + getGroupsCountAndUsername(user_name,group_names));
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
Partition ret = null;
Exception ex = null;
@@ -5592,7 +5645,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
String dbName = parsedDbName[DB_NAME];
String tableName = tbl_name.toLowerCase();
- startTableFunction("get_partitions_pspec", catName, dbName, tableName);
+ startPartitionFunction("get_partitions_pspec", catName, dbName, tableName, max_parts);
List<PartitionSpec> partitionSpecs = null;
try {
@@ -5678,7 +5731,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
public List<String> get_partition_names(final String db_name, final String tbl_name,
final short max_parts) throws NoSuchObjectException, MetaException {
String[] parsedDbName = parseDbName(db_name, conf);
- startTableFunction("get_partition_names", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
+ startPartitionFunction("get_partition_names", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, max_parts);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
List<String> ret = null;
Exception ex = null;
@@ -5703,7 +5756,9 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
String catName = request.isSetCatName() ? request.getCatName() : getDefaultCatalog(conf);
String dbName = request.getDbName();
String tblName = request.getTblName();
-
+ long maxParts = request.getMaxParts();
+ String filter = request.isSetFilter() ? request.getFilter() : "";
+ startPartitionFunction("get_partition_values", catName, dbName, tblName, (int) maxParts, filter);
try {
authorizeTableForPartitionMetadata(catName, dbName, tblName);
@@ -6562,8 +6617,8 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
final String tbl_name, final List<String> part_vals,
final short max_parts) throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
- startPartitionFunction("get_partitions_ps", parsedDbName[CAT_NAME], parsedDbName[DB_NAME],
- tbl_name, part_vals);
+ startPartitionFunction("get_partitions_ps", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name, max_parts,
+ part_vals);
List<Partition> ret = null;
Exception ex = null;
@@ -6655,8 +6710,8 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
final String tbl_name, final List<String> part_vals, final short max_parts)
throws TException {
String[] parsedDbName = parseDbName(db_name, conf);
- startPartitionFunction("get_partitions_names_ps", parsedDbName[CAT_NAME],
- parsedDbName[DB_NAME], tbl_name, part_vals);
+ startPartitionFunction("get_partitions_names_ps", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name,
+ max_parts, part_vals);
fireReadTablePreEvent(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tbl_name);
List<String> ret = null;
Exception ex = null;
@@ -6847,8 +6902,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
String catName = request.isSetCatName() ? request.getCatName().toLowerCase() : getDefaultCatalog(conf);
String dbName = request.getDbName().toLowerCase();
String tblName = request.getTblName().toLowerCase();
- startFunction("get_partitions_statistics_req", ": table=" +
- TableName.getQualified(catName, dbName, tblName));
+ startPartitionFunction("get_partitions_statistics_req", catName, dbName, tblName, request.getPartNames());
PartitionsStatsResult result = null;
List<String> lowerCaseColNames = new ArrayList<>(request.getColNames().size());
@@ -7264,8 +7318,8 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
throws TException {
String[] parsedDbName = parseDbName(dbName, conf);
- startTableFunction("get_partitions_by_filter_pspec", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
-
+ startPartitionFunction("get_partitions_by_filter_pspec", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName,
+ maxParts, filter);
List<PartitionSpec> partitionSpecs = null;
try {
Table table = get_table_core(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName);
@@ -7330,7 +7384,10 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
PartitionsByExprRequest req) throws TException {
String dbName = req.getDbName(), tblName = req.getTblName();
String catName = req.isSetCatName() ? req.getCatName() : getDefaultCatalog(conf);
- startTableFunction("get_partitions_by_expr", catName, dbName, tblName);
+ String expr = req.isSetExpr() ? Arrays.toString((req.getExpr())) : "";
+ String defaultPartitionName = req.isSetDefaultPartitionName() ? req.getDefaultPartitionName() : "";
+ int maxParts = req.getMaxParts();
+ startPartitionFunction("get_partitions_by_expr", catName, dbName, tblName, maxParts, expr, defaultPartitionName);
fireReadTablePreEvent(catName, dbName, tblName);
PartitionsByExprResult ret = null;
Exception ex = null;
@@ -7363,8 +7420,9 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
if (parsedDbName[DB_NAME] == null || tblName == null) {
throw new MetaException("The DB and table name cannot be null.");
}
- startTableFunction("get_num_partitions_by_filter", parsedDbName[CAT_NAME],
- parsedDbName[DB_NAME], tblName);
+ startFunction("get_num_partitions_by_filter",
+ " : tbl=" + TableName.getQualified(parsedDbName[CAT_NAME], parsedDbName[DB_NAME], tblName) + " Filter="
+ + filter);
int ret = -1;
Exception ex = null;