You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/08/30 12:08:07 UTC
[impala] 05/06: IMPALA-11401,IMPALA-10794: Add logs and thread names for catalogd RPCs
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch branch-4.1.1
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 14b2d414f44da74335c60e6c38f173e71eea20d9
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Fri Jul 22 15:34:05 2022 +0800
IMPALA-11401,IMPALA-10794: Add logs and thread names for catalogd RPCs
We've seen catalogd throws OutOfMemoryError when serializing large
responses (i.e. size > 2GB). However, the related table names are
missing in the logs. Admins would like to get the table names and
blacklist those tables until they are optimized (e.g. reducing
partitions).
To improve the supportability, this patch adds logs in the Catalogd RPC
code paths to log some details of the request, also add thread
annotations to improve readability of jstacks.
Tests:
- Add unit tests for short descriptions of requests.
- Manually add codes to throw OutOfMemoryError and verify the logs
shown as expected.
- Run test_concurrent_ddls.py and metadata tests. Capture jstacks and
verify the thread annotations are shown.
- Run CORE tests
Change-Id: Iac7f2eda8b95643a3d3c3bef64ea71b67b20595a
Reviewed-on: http://gerrit.cloudera.org:8080/18772
Reviewed-by: Csaba Ringhofer <cs...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-on: http://gerrit.cloudera.org:8080/18916
Tested-by: Quanlong Huang <hu...@gmail.com>
---
.../org/apache/impala/analysis/ColumnName.java | 6 +
.../org/apache/impala/analysis/FunctionName.java | 4 +
.../apache/impala/analysis/ResetMetadataStmt.java | 5 +
.../java/org/apache/impala/analysis/TableName.java | 4 +
.../impala/catalog/local/CatalogdMetaProvider.java | 4 +-
.../apache/impala/service/CatalogOpExecutor.java | 11 +-
.../java/org/apache/impala/service/JniCatalog.java | 146 +++++++++++---
.../java/org/apache/impala/util/CatalogOpUtil.java | 148 ++++++++++++++
.../org/apache/impala/util/CatalogOpUtilTest.java | 221 +++++++++++++++++++++
9 files changed, 513 insertions(+), 36 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/ColumnName.java b/fe/src/main/java/org/apache/impala/analysis/ColumnName.java
index 8eb285bce..41d50a1ab 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ColumnName.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ColumnName.java
@@ -18,6 +18,7 @@
package org.apache.impala.analysis;
import com.google.common.base.Preconditions;
+import org.apache.impala.thrift.TColumnName;
/**
* Represents a column name that optionally includes its database name.
@@ -36,4 +37,9 @@ public class ColumnName {
public TableName getTableName() { return tableName_; }
public String getColumnName() { return columnName_; }
+
+ public static String thriftToString(TColumnName colName) {
+ return TableName.thriftToString(colName.getTable_name()) + "." +
+ colName.getColumn_name();
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/FunctionName.java b/fe/src/main/java/org/apache/impala/analysis/FunctionName.java
index 288f7cb6e..b3093c0c1 100644
--- a/fe/src/main/java/org/apache/impala/analysis/FunctionName.java
+++ b/fe/src/main/java/org/apache/impala/analysis/FunctionName.java
@@ -161,4 +161,8 @@ public class FunctionName {
public static FunctionName fromThrift(TFunctionName fnName) {
return new FunctionName(fnName.getDb_name(), fnName.getFunction_name());
}
+
+ public static String thriftToString(TFunctionName fnName) {
+ return fromThrift(fnName).toString();
+ }
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
index c892a2e65..87d4629f7 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
@@ -126,6 +126,11 @@ public class ResetMetadataStmt extends StatementBase {
@VisibleForTesting
protected Action getAction() { return action_; }
+ @VisibleForTesting
+ public void setRequestingUser(User user) {
+ requestingUser_ = user;
+ }
+
@Override
public void collectTableRefs(List<TableRef> tblRefs) {
if (tableName_ != null && partitionSpec_ != null) {
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableName.java b/fe/src/main/java/org/apache/impala/analysis/TableName.java
index ae6426e33..e413cca2b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableName.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableName.java
@@ -126,6 +126,10 @@ public class TableName {
return new TableName(tableName.getDb_name(), tableName.getTable_name());
}
+ public static String thriftToString(TTableName tableName) {
+ return fromThrift(tableName).toString();
+ }
+
public TTableName toThrift() { return new TTableName(db_, tbl_); }
/**
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index 794124f44..4c4ec1091 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -97,6 +97,7 @@ import org.apache.impala.thrift.TValidWriteIdList;
import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.IcebergUtil;
import org.apache.impala.util.ListMap;
+import org.apache.impala.util.ThreadNameAnnotator;
import org.apache.thrift.TDeserializer;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
@@ -511,7 +512,8 @@ public class CatalogdMetaProvider implements MetaProvider {
Stopwatch sw = Stopwatch.createStarted();
boolean hit = false;
boolean isPiggybacked = false;
- try {
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(
+ "LoadWithCaching for " + itemString)) {
CompletableFuture<Object> f = new CompletableFuture<Object>();
// NOTE: the Cache ensures that this is an atomic operation of either returning
// an existing value or inserting our own. Only one thread can think it is the
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 76ac5e6f7..43c9a50a1 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -230,6 +230,7 @@ import org.apache.impala.thrift.TUpdateCatalogResponse;
import org.apache.impala.thrift.TUpdatedPartition;
import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.AcidUtils.TblTransaction;
+import org.apache.impala.util.CatalogOpUtil;
import org.apache.impala.util.CompressionUtil;
import org.apache.impala.util.DebugUtils;
import org.apache.impala.util.HdfsCachingUtil;
@@ -237,6 +238,7 @@ import org.apache.impala.util.IcebergUtil;
import org.apache.impala.util.KuduUtil;
import org.apache.impala.util.MetaStoreUtil;
import org.apache.impala.util.MetaStoreUtil.TableInsertEventInfo;
+import org.apache.impala.util.ThreadNameAnnotator;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -5458,7 +5460,10 @@ public class CatalogOpExecutor {
// Add partitions to metastore.
Map<String, Long> partitionToEventId = Maps.newHashMap();
- try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
+ String annotation = String.format("Recovering %d partitions for %s",
+ hmsPartitions.size(), tbl.getFullName());
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(annotation);
+ MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
List<Partition> addedPartitions = addHmsPartitions(msClient, tbl, hmsPartitions,
partitionToEventId, true);
addHdfsPartitions(msClient, tbl, addedPartitions, partitionToEventId);
@@ -5997,9 +6002,7 @@ public class CatalogOpExecutor {
*/
public TResetMetadataResponse execResetMetadata(TResetMetadataRequest req)
throws CatalogException {
- String cmdString = String.format("%s issued by %s",
- req.is_refresh ? "REFRESH":"INVALIDATE",
- req.header != null ? req.header.requesting_user : " unknown user");
+ String cmdString = CatalogOpUtil.getShortDescForReset(req);
TResetMetadataResponse resp = new TResetMetadataResponse();
resp.setResult(new TCatalogUpdateResult());
resp.getResult().setCatalog_service_id(JniCatalog.getServiceId());
diff --git a/fe/src/main/java/org/apache/impala/service/JniCatalog.java b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
index e8f2951f4..126d08f63 100644
--- a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
+++ b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.AuthorizationFactory;
import org.apache.impala.authorization.AuthorizationManager;
+import org.apache.impala.catalog.Catalog;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.CatalogServiceCatalog;
import org.apache.impala.catalog.Db;
@@ -46,6 +47,7 @@ import org.apache.impala.catalog.metastore.ICatalogMetastoreServer;
import org.apache.impala.catalog.metastore.NoOpCatalogMetastoreServer;
import org.apache.impala.catalog.monitor.CatalogMonitor;
import org.apache.impala.catalog.monitor.CatalogOperationMetrics;
+import org.apache.impala.common.PrintUtils;
import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
@@ -78,8 +80,10 @@ import org.apache.impala.thrift.TUpdateCatalogRequest;
import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.thrift.TUpdateTableUsageRequest;
import org.apache.impala.util.AuthorizationUtil;
+import org.apache.impala.util.CatalogOpUtil;
import org.apache.impala.util.GlogAppender;
import org.apache.impala.util.PatternMatcher;
+import org.apache.impala.util.ThreadNameAnnotator;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.apache.thrift.protocol.TBinaryProtocol;
@@ -219,11 +223,20 @@ public class JniCatalog {
long start = System.currentTimeMillis();
TGetCatalogDeltaRequest params = new TGetCatalogDeltaRequest();
JniUtil.deserializeThrift(protocolFactory_, params, thriftGetCatalogDeltaReq);
- byte[] res = new TSerializer(protocolFactory_).serialize(new TGetCatalogDeltaResponse(
- catalog_.getCatalogDelta(params.getNative_catalog_server_ptr(),
- params.getFrom_version())));
- JniUtil.logResponse(res.length, start, params, "getCatalogDelta");
- return res;
+ TSerializer serializer = new TSerializer(protocolFactory_);
+ String shortDesc = "getting catalog delta from version " + params.getFrom_version();
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(shortDesc)) {
+ byte[] res = serializer.serialize(new TGetCatalogDeltaResponse(
+ catalog_.getCatalogDelta(params.getNative_catalog_server_ptr(),
+ params.getFrom_version())));
+ JniUtil.logResponse(res.length, start, params, "getCatalogDelta");
+ return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in {}. Time spent: {}.",
+ shortDesc, PrintUtils.printTimeMs(duration));
+ throw e;
+ }
}
/**
@@ -236,17 +249,25 @@ public class JniCatalog {
/**
* Executes the given DDL request and returns the result.
*/
- public byte[] execDdl(byte[] thriftDdlExecReq) throws ImpalaException {
+ public byte[] execDdl(byte[] thriftDdlExecReq) throws ImpalaException, TException {
long start = System.currentTimeMillis();
TDdlExecRequest params = new TDdlExecRequest();
JniUtil.deserializeThrift(protocolFactory_, params, thriftDdlExecReq);
TSerializer serializer = new TSerializer(protocolFactory_);
- try {
+ String shortDesc = CatalogOpUtil.getShortDescForExecDdl(params);
+ LOG.info("execDdl request: " + shortDesc);
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(shortDesc)) {
byte[] res = serializer.serialize(catalogOpExecutor_.execDdlRequest(params));
JniUtil.logResponse(res.length, start, params, "execDdl");
+ long duration = System.currentTimeMillis() - start;
+ LOG.info("finished execDdl request: {}. Time spent: {}",
+ shortDesc, PrintUtils.printTimeMs(duration));
return res;
- } catch (TException e) {
- throw new InternalException(e.getMessage());
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in execDdl for {}. Time spent: {}.",
+ shortDesc, PrintUtils.printTimeMs(duration));
+ throw e;
}
}
@@ -260,10 +281,20 @@ public class JniCatalog {
JniUtil.deserializeThrift(protocolFactory_, req, thriftResetMetadataReq);
TSerializer serializer = new TSerializer(protocolFactory_);
catalogOperationUsage.increment(req);
- try {
+ String shortDesc = CatalogOpUtil.getShortDescForReset(req);
+ LOG.info("resetMetadata request: " + shortDesc);
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(shortDesc)) {
byte[] res = serializer.serialize(catalogOpExecutor_.execResetMetadata(req));
JniUtil.logResponse(res.length, start, req, "resetMetadata");
+ long duration = System.currentTimeMillis() - start;
+ LOG.info("finished resetMetadata request: {}. Time spent: {}",
+ shortDesc, PrintUtils.printTimeMs(duration));
return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in resetMetadata for {}. Time spent: {}.",
+ shortDesc, PrintUtils.printTimeMs(duration));
+ throw e;
} finally {
catalogOperationUsage.decrement(req);
}
@@ -317,9 +348,17 @@ public class JniCatalog {
long start = System.currentTimeMillis();
TGetTableMetricsParams params = new TGetTableMetricsParams();
JniUtil.deserializeThrift(protocolFactory_, params, getTableMetricsParams);
- String res = catalog_.getTableMetrics(params.table_name);
- JniUtil.logResponse(res.length(), start, params, "getTableMetrics");
- return res;
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(
+ "getTableMetrics " + params.table_name)) {
+ String res = catalog_.getTableMetrics(params.table_name);
+ JniUtil.logResponse(res.length(), start, params, "getTableMetrics");
+ return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in getTableMetrics {}. Time spent: {}.", params.table_name,
+ PrintUtils.printTimeMs(duration));
+ throw e;
+ }
}
/**
@@ -328,12 +367,21 @@ public class JniCatalog {
public byte[] getCatalogObject(byte[] thriftParams) throws ImpalaException,
TException {
long start = System.currentTimeMillis();
- TCatalogObject objectDescription = new TCatalogObject();
- JniUtil.deserializeThrift(protocolFactory_, objectDescription, thriftParams);
+ TCatalogObject objectDesc = new TCatalogObject();
+ JniUtil.deserializeThrift(protocolFactory_, objectDesc, thriftParams);
TSerializer serializer = new TSerializer(protocolFactory_);
- byte[] res = serializer.serialize(catalog_.getTCatalogObject(objectDescription));
- JniUtil.logResponse(res.length, start, objectDescription, "getCatalogObject");
- return res;
+ String shortDesc = "getting thrift catalog object of "
+ + Catalog.toCatalogObjectKey(objectDesc);
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(shortDesc)) {
+ byte[] res = serializer.serialize(catalog_.getTCatalogObject(objectDesc));
+ JniUtil.logResponse(res.length, start, objectDesc, "getCatalogObject");
+ return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in {}. Time spent: {}.", shortDesc,
+ PrintUtils.printTimeMs(duration));
+ throw e;
+ }
}
/**
@@ -343,12 +391,21 @@ public class JniCatalog {
public String getJsonCatalogObject(byte[] thriftParams) throws ImpalaException,
TException {
long start = System.currentTimeMillis();
- TCatalogObject objectDescription = new TCatalogObject();
- JniUtil.deserializeThrift(protocolFactory_, objectDescription, thriftParams);
+ TCatalogObject objectDesc = new TCatalogObject();
+ JniUtil.deserializeThrift(protocolFactory_, objectDesc, thriftParams);
TSerializer jsonSerializer = new TSerializer(new TSimpleJSONProtocol.Factory());
- String res = jsonSerializer.toString(catalog_.getTCatalogObject(objectDescription));
- JniUtil.logResponse(res.length(), start, objectDescription, "getJsonCatalogObject");
- return res;
+ String shortDesc = "getting json catalog object of "
+ + Catalog.toCatalogObjectKey(objectDesc);
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(shortDesc)) {
+ String res = jsonSerializer.toString(catalog_.getTCatalogObject(objectDesc));
+ JniUtil.logResponse(res.length(), start, objectDesc, "getJsonCatalogObject");
+ return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in {}. Time spent: {}.", shortDesc,
+ PrintUtils.printTimeMs(duration));
+ throw e;
+ }
}
public byte[] getPartialCatalogObject(byte[] thriftParams) throws ImpalaException,
@@ -358,9 +415,16 @@ public class JniCatalog {
new TGetPartialCatalogObjectRequest();
JniUtil.deserializeThrift(protocolFactory_, req, thriftParams);
TSerializer serializer = new TSerializer(protocolFactory_);
- byte[] res = serializer.serialize(catalog_.getPartialCatalogObject(req));
- JniUtil.logResponse(res.length, start, req, "getPartialCatalogObject");
- return res;
+ try {
+ byte[] res = serializer.serialize(catalog_.getPartialCatalogObject(req));
+ JniUtil.logResponse(res.length, start, req, "getPartialCatalogObject");
+ return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in getting PartialCatalogObject of {}. Time spent: {}.",
+ Catalog.toCatalogObjectKey(req.object_desc), PrintUtils.printTimeMs(duration));
+ throw e;
+ }
}
/**
@@ -405,15 +469,24 @@ public class JniCatalog {
JniUtil.deserializeThrift(protocolFactory_, request, thriftParams);
TSerializer serializer = new TSerializer(protocolFactory_);
TGetPartitionStatsResponse response = new TGetPartitionStatsResponse();
- try {
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(
+ "Getting partition stats of " + request.table_name)) {
response.setPartition_stats(catalog_.getPartitionStats(request));
} catch (CatalogException e) {
response.setStatus(
new TStatus(TErrorCode.INTERNAL_ERROR, ImmutableList.of(e.getMessage())));
}
- byte[] res = serializer.serialize(response);
- JniUtil.logResponse(res.length, start, request, "getPartitionStats");
- return res;
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(
+ "Serializing partition stats of " + request.table_name)) {
+ byte[] res = serializer.serialize(response);
+ JniUtil.logResponse(res.length, start, request, "getPartitionStats");
+ return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in serializing partition stats of {}. Time spent in method: {}.",
+ request.table_name, PrintUtils.printTimeMs(duration));
+ throw e;
+ }
}
/**
@@ -427,10 +500,21 @@ public class JniCatalog {
JniUtil.deserializeThrift(protocolFactory_, request, thriftUpdateCatalog);
TSerializer serializer = new TSerializer(protocolFactory_);
catalogOperationUsage.increment(request);
- try {
+ String shortDesc = String.format("updateCatalog for %s.%s",
+ request.db_name, request.target_table);
+ LOG.info(shortDesc);
+ try (ThreadNameAnnotator tna = new ThreadNameAnnotator(shortDesc)) {
byte[] res = serializer.serialize(catalogOpExecutor_.updateCatalog(request));
JniUtil.logResponse(res.length, start, request, "updateCatalog");
+ long duration = System.currentTimeMillis() - start;
+ LOG.info("finished {}. Time spent: {}", shortDesc,
+ PrintUtils.printTimeMs(duration));
return res;
+ } catch (Throwable e) {
+ long duration = System.currentTimeMillis() - start;
+ LOG.error("Error in {}. Time spent: {}.", shortDesc,
+ PrintUtils.printTimeMs(duration));
+ throw e;
} finally {
catalogOperationUsage.decrement(request);
}
diff --git a/fe/src/main/java/org/apache/impala/util/CatalogOpUtil.java b/fe/src/main/java/org/apache/impala/util/CatalogOpUtil.java
new file mode 100644
index 000000000..0b96a3ecb
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/util/CatalogOpUtil.java
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.util;
+
+import org.apache.impala.analysis.ColumnName;
+import org.apache.impala.analysis.FunctionName;
+import org.apache.impala.analysis.TableName;
+import org.apache.impala.thrift.TCommentOnParams;
+import org.apache.impala.thrift.TDdlExecRequest;
+import org.apache.impala.thrift.TResetMetadataRequest;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CatalogOpUtil {
+ private static final Logger LOG = LoggerFactory.getLogger(CatalogOpUtil.class);
+
+ /**
+ * Get a short description for the execDdl request.
+ */
+ public static String getShortDescForExecDdl(TDdlExecRequest req) {
+ String target;
+ try {
+ switch (req.ddl_type) {
+ case ALTER_DATABASE:
+ target = req.getAlter_db_params().getDb();
+ break;
+ case ALTER_TABLE:
+ target = TableName.thriftToString(req.getAlter_table_params().getTable_name());
+ break;
+ case ALTER_VIEW:
+ target = TableName.thriftToString(req.getAlter_view_params().getView_name());
+ break;
+ case CREATE_DATABASE:
+ target = req.getCreate_db_params().getDb();
+ break;
+ case CREATE_TABLE_AS_SELECT:
+ case CREATE_TABLE:
+ target = TableName.thriftToString(req.getCreate_table_params().getTable_name());
+ break;
+ case CREATE_TABLE_LIKE:
+ target = TableName.thriftToString(
+ req.getCreate_table_like_params().getTable_name());
+ break;
+ case CREATE_VIEW:
+ target = TableName.thriftToString(req.getCreate_view_params().getView_name());
+ break;
+ case CREATE_FUNCTION:
+ target = FunctionName.thriftToString(
+ req.getCreate_fn_params().getFn().getName());
+ break;
+ case COMMENT_ON: {
+ TCommentOnParams params = req.getComment_on_params();
+ if (params.isSetDb()) {
+ target = "DB " + params.getDb();
+ } else if (params.isSetTable_name()) {
+ target = "TABLE " + TableName.thriftToString(params.getTable_name());
+ } else if (params.isSetColumn_name()) {
+ target = "COLUMN " + ColumnName.thriftToString(params.getColumn_name());
+ } else {
+ target = "";
+ }
+ break;
+ }
+ case DROP_STATS:
+ target = TableName.thriftToString(req.getDrop_stats_params().getTable_name());
+ break;
+ case DROP_DATABASE:
+ target = req.getDrop_db_params().getDb();
+ break;
+ case DROP_TABLE:
+ case DROP_VIEW:
+ target = TableName.thriftToString(
+ req.getDrop_table_or_view_params().getTable_name());
+ break;
+ case TRUNCATE_TABLE:
+ target = TableName.thriftToString(req.getTruncate_params().getTable_name());
+ break;
+ case DROP_FUNCTION:
+ target = FunctionName.thriftToString(req.getDrop_fn_params().fn_name);
+ break;
+ case CREATE_ROLE:
+ case DROP_ROLE:
+ target = req.getCreate_drop_role_params().getRole_name();
+ break;
+ case GRANT_ROLE:
+ case REVOKE_ROLE:
+ target = req.getGrant_revoke_role_params().getRole_names() + " GROUP " +
+ req.getGrant_revoke_role_params().getGroup_names();
+ break;
+ case GRANT_PRIVILEGE:
+ target = "TO " + req.getGrant_revoke_priv_params().getPrincipal_name();
+ break;
+ case REVOKE_PRIVILEGE:
+ target = "FROM " + req.getGrant_revoke_priv_params().getPrincipal_name();
+ break;
+ default:
+ target = "";
+ }
+ } catch (Throwable t) {
+ // This method is used for all DDL RPCs. We should not fail them by errors happen
+ // here. Catch all exceptions and just log the error.
+ target = "unknown target";
+ LOG.error("Failed to get the target for request", t);
+ }
+
+ String user = "unknown user";
+ if (req.isSetHeader() && req.header.isSetRequesting_user()) {
+ user = req.header.requesting_user;
+ }
+ return String.format("%s%s issued by %s", req.ddl_type, " " + target, user);
+ }
+
+ /**
+ * Get a short description for the resetMetadata request.
+ */
+ public static String getShortDescForReset(TResetMetadataRequest req) {
+ String cmd = req.is_refresh ? "REFRESH " : "INVALIDATE ";
+ if (req.isSetDb_name()) {
+ if (req.is_refresh) cmd += "FUNCTIONS IN ";
+ cmd += "DATABASE " + req.getDb_name();
+ } else if (req.isSetTable_name()) {
+ cmd += "TABLE " + TableName.fromThrift(req.getTable_name());
+ if (req.isSetPartition_spec()) cmd += " PARTITIONS";
+ } else if (req.isAuthorization()) {
+ cmd += "AUTHORIZATION";
+ } else {
+ // Global INVALIDATE METADATA
+ cmd += "ALL";
+ }
+ String user = req.header != null ? req.header.requesting_user : "unknown user";
+ return String.format("%s issued by %s", cmd, user);
+ }
+}
diff --git a/fe/src/test/java/org/apache/impala/util/CatalogOpUtilTest.java b/fe/src/test/java/org/apache/impala/util/CatalogOpUtilTest.java
new file mode 100644
index 000000000..11d8a9f62
--- /dev/null
+++ b/fe/src/test/java/org/apache/impala/util/CatalogOpUtilTest.java
@@ -0,0 +1,221 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.util;
+
+import org.apache.impala.analysis.GrantRevokePrivStmt;
+import org.apache.impala.analysis.GrantRevokeRoleStmt;
+import org.apache.impala.analysis.PrivilegeSpec;
+import org.apache.impala.analysis.ResetMetadataStmt;
+import org.apache.impala.analysis.TableName;
+import org.apache.impala.authorization.User;
+import org.apache.impala.thrift.TAlterDbParams;
+import org.apache.impala.thrift.TAlterDbType;
+import org.apache.impala.thrift.TAlterTableParams;
+import org.apache.impala.thrift.TColumnName;
+import org.apache.impala.thrift.TCommentOnParams;
+import org.apache.impala.thrift.TCreateDbParams;
+import org.apache.impala.thrift.TCreateDropRoleParams;
+import org.apache.impala.thrift.TCreateFunctionParams;
+import org.apache.impala.thrift.TCreateOrAlterViewParams;
+import org.apache.impala.thrift.TCreateTableParams;
+import org.apache.impala.thrift.TDdlExecRequest;
+import org.apache.impala.thrift.TDdlQueryOptions;
+import org.apache.impala.thrift.TDdlType;
+import org.apache.impala.thrift.TDropDbParams;
+import org.apache.impala.thrift.TDropFunctionParams;
+import org.apache.impala.thrift.TDropTableOrViewParams;
+import org.apache.impala.thrift.TFunction;
+import org.apache.impala.thrift.TFunctionName;
+import org.apache.impala.thrift.TGrantRevokePrivParams;
+import org.apache.impala.thrift.TGrantRevokeRoleParams;
+import org.apache.impala.thrift.TPrincipalType;
+import org.apache.impala.thrift.TPrivilegeLevel;
+import org.apache.impala.thrift.TResetMetadataRequest;
+import org.apache.impala.thrift.TTableName;
+import org.junit.Test;
+
+import java.util.Collections;
+
+import static org.junit.Assert.assertEquals;
+
+public class CatalogOpUtilTest {
+
+ private void testResetStmt(ResetMetadataStmt stmt, User user, String expected)
+ throws Exception {
+ stmt.setRequestingUser(user);
+ assertEquals(expected, CatalogOpUtil.getShortDescForReset(stmt.toThrift()));
+ }
+
+ @Test
+ public void testResetMetadataDesc() throws Exception {
+ User user = new User("Alice");
+ TableName tblName = new TableName("default", "tbl");
+
+ testResetStmt(ResetMetadataStmt.createInvalidateStmt(),
+ user, "INVALIDATE ALL issued by Alice");
+ testResetStmt(ResetMetadataStmt.createInvalidateStmt(tblName),
+ user, "INVALIDATE TABLE default.tbl issued by Alice");
+ testResetStmt(ResetMetadataStmt.createRefreshTableStmt(tblName),
+ user, "REFRESH TABLE default.tbl issued by Alice");
+ testResetStmt(ResetMetadataStmt.createRefreshFunctionsStmt("db1"),
+ user, "REFRESH FUNCTIONS IN DATABASE db1 issued by Alice");
+ testResetStmt(ResetMetadataStmt.createRefreshAuthorizationStmt(),
+ user, "REFRESH AUTHORIZATION issued by Alice");
+
+ // Test REFRESH PARTITIONS using a fake partition spec
+ ResetMetadataStmt stmt = ResetMetadataStmt.createRefreshTableStmt(tblName);
+ stmt.setRequestingUser(user);
+ TResetMetadataRequest req = stmt.toThrift();
+ req.setPartition_spec(Collections.emptyList());
+ assertEquals("REFRESH TABLE default.tbl PARTITIONS issued by Alice",
+ CatalogOpUtil.getShortDescForReset(req));
+ }
+
+ @Test
+ public void testDdlDesc() {
+ TDdlExecRequest req;
+ TTableName tblName = new TTableName("db1", "tbl1");
+
+ req = new TDdlExecRequest();
+ req.setQuery_options(new TDdlQueryOptions());
+ req.setDdl_type(TDdlType.CREATE_DATABASE);
+ TCreateDbParams createDbParams = new TCreateDbParams();
+ createDbParams.setDb("db1");
+ req.setCreate_db_params(createDbParams);
+ assertEquals("CREATE_DATABASE db1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.DROP_DATABASE);
+ TDropDbParams dropDbParams = new TDropDbParams();
+ dropDbParams.setDb("db1");
+ req.setDrop_db_params(dropDbParams);
+ assertEquals("DROP_DATABASE db1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.ALTER_DATABASE);
+ TAlterDbParams alterDbParams = new TAlterDbParams();
+ alterDbParams.setAlter_type(TAlterDbType.SET_OWNER);
+ alterDbParams.setDb("db1");
+ req.setAlter_db_params(alterDbParams);
+ assertEquals("ALTER_DATABASE db1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.CREATE_TABLE);
+ TCreateTableParams createTableParams = new TCreateTableParams();
+ createTableParams.setTable_name(tblName);
+ req.setCreate_table_params(createTableParams);
+ assertEquals("CREATE_TABLE db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ req.setDdl_type(TDdlType.CREATE_TABLE_AS_SELECT);
+ assertEquals("CREATE_TABLE_AS_SELECT db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.ALTER_TABLE);
+ TAlterTableParams alterTableParams = new TAlterTableParams();
+ alterTableParams.setTable_name(tblName);
+ req.setAlter_table_params(alterTableParams);
+ assertEquals("ALTER_TABLE db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.CREATE_VIEW);
+ TCreateOrAlterViewParams alterViewParams = new TCreateOrAlterViewParams();
+ alterViewParams.setView_name(tblName);
+ req.setCreate_view_params(alterViewParams);
+ assertEquals("CREATE_VIEW db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ req.setDdl_type(TDdlType.ALTER_VIEW);
+ req.setAlter_view_params(alterViewParams);
+ assertEquals("ALTER_VIEW db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.DROP_TABLE);
+ TDropTableOrViewParams dropTableOrViewParams = new TDropTableOrViewParams();
+ dropTableOrViewParams.setTable_name(tblName);
+ req.setDrop_table_or_view_params(dropTableOrViewParams);
+ assertEquals("DROP_TABLE db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ req.setDdl_type(TDdlType.DROP_VIEW);
+ assertEquals("DROP_VIEW db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.COMMENT_ON);
+ TCommentOnParams commentOnParams = new TCommentOnParams();
+ commentOnParams.setDb("db1");
+ req.setComment_on_params(commentOnParams);
+ assertEquals("COMMENT_ON DB db1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ commentOnParams.clear();
+ commentOnParams.setTable_name(tblName);
+ req.setComment_on_params(commentOnParams);
+ assertEquals("COMMENT_ON TABLE db1.tbl1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ commentOnParams.clear();
+ commentOnParams.setColumn_name(new TColumnName(tblName, "col1"));
+ req.setComment_on_params(commentOnParams);
+ assertEquals("COMMENT_ON COLUMN db1.tbl1.col1 issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.CREATE_FUNCTION);
+ TCreateFunctionParams createFunctionParams = new TCreateFunctionParams();
+ TFunction fn = new TFunction();
+ fn.setName(new TFunctionName("my_func"));
+ createFunctionParams.setFn(fn);
+ req.setCreate_fn_params(createFunctionParams);
+ assertEquals("CREATE_FUNCTION my_func issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.DROP_FUNCTION);
+ TDropFunctionParams dropFunctionParams = new TDropFunctionParams();
+ dropFunctionParams.setFn_name(new TFunctionName("my_func"));
+ req.setDrop_fn_params(dropFunctionParams);
+ assertEquals("DROP_FUNCTION my_func issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.CREATE_ROLE);
+ TCreateDropRoleParams createDropRoleParams = new TCreateDropRoleParams();
+ createDropRoleParams.setRole_name("my_role");
+ req.setCreate_drop_role_params(createDropRoleParams);
+ assertEquals("CREATE_ROLE my_role issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ req.setDdl_type(TDdlType.DROP_ROLE);
+ assertEquals("DROP_ROLE my_role issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.GRANT_ROLE);
+ TGrantRevokeRoleParams grantRevokeRoleParams;
+ grantRevokeRoleParams = new GrantRevokeRoleStmt("my_role", "my_group", true)
+ .toThrift();
+ req.setGrant_revoke_role_params(grantRevokeRoleParams);
+ assertEquals("GRANT_ROLE [my_role] GROUP [my_group] issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ req.setDdl_type(TDdlType.REVOKE_ROLE);
+ assertEquals("REVOKE_ROLE [my_role] GROUP [my_group] issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+
+ req.setDdl_type(TDdlType.GRANT_PRIVILEGE);
+ TGrantRevokePrivParams grantRevokePrivParams = new GrantRevokePrivStmt("my_role",
+ PrivilegeSpec.createServerScopedPriv(TPrivilegeLevel.SELECT), true, false,
+ TPrincipalType.ROLE).toThrift();
+ req.setGrant_revoke_priv_params(grantRevokePrivParams);
+ assertEquals("GRANT_PRIVILEGE TO my_role issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ req.setDdl_type(TDdlType.REVOKE_PRIVILEGE);
+ assertEquals("REVOKE_PRIVILEGE FROM my_role issued by unknown user",
+ CatalogOpUtil.getShortDescForExecDdl(req));
+ }
+}