You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/05/08 09:25:47 UTC
[incubator-doris] branch master updated: [feature] (sql-digest) support sql digest (#8919)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new c633402ce3 [feature] (sql-digest) support sql digest (#8919)
c633402ce3 is described below
commit c633402ce3753f3726ec0bbbe3b966f8e998625b
Author: Henry2SS <45...@users.noreply.github.com>
AuthorDate: Sun May 8 17:25:41 2022 +0800
[feature] (sql-digest) support sql digest (#8919)
---
docs/en/ecosystem/audit-plugin.md | 3 +-
docs/zh-CN/ecosystem/audit-plugin.md | 1 +
.../org/apache/doris/analysis/AnalyticExpr.java | 41 ++++++
.../org/apache/doris/analysis/AnalyticWindow.java | 26 ++++
.../org/apache/doris/analysis/ArithmeticExpr.java | 9 ++
.../org/apache/doris/analysis/ArrayLiteral.java | 8 ++
.../apache/doris/analysis/BetweenPredicate.java | 9 +-
.../org/apache/doris/analysis/BinaryPredicate.java | 5 +
.../java/org/apache/doris/analysis/CaseExpr.java | 18 +++
.../java/org/apache/doris/analysis/CastExpr.java | 21 +++
.../apache/doris/analysis/CompoundPredicate.java | 9 ++
.../org/apache/doris/analysis/ExistsPredicate.java | 13 ++
.../main/java/org/apache/doris/analysis/Expr.java | 21 +++
.../java/org/apache/doris/analysis/FromClause.java | 11 ++
.../apache/doris/analysis/FunctionCallExpr.java | 55 ++++++++
.../org/apache/doris/analysis/InPredicate.java | 13 ++
.../org/apache/doris/analysis/InlineViewRef.java | 20 +++
.../org/apache/doris/analysis/IsNullPredicate.java | 5 +
.../org/apache/doris/analysis/LikePredicate.java | 6 +
.../org/apache/doris/analysis/LimitElement.java | 12 ++
.../org/apache/doris/analysis/LiteralExpr.java | 5 +
.../org/apache/doris/analysis/OrderByElement.java | 16 +++
.../org/apache/doris/analysis/OutFileClause.java | 9 ++
.../java/org/apache/doris/analysis/QueryStmt.java | 4 +
.../org/apache/doris/analysis/SelectListItem.java | 14 ++
.../java/org/apache/doris/analysis/SelectStmt.java | 77 +++++++++++
.../apache/doris/analysis/SetOperationStmt.java | 51 ++++++++
.../java/org/apache/doris/analysis/Subquery.java | 5 +
.../java/org/apache/doris/analysis/TableRef.java | 24 ++++
.../doris/analysis/TimestampArithmeticExpr.java | 38 ++++++
.../java/org/apache/doris/analysis/WithClause.java | 15 +++
.../java/org/apache/doris/plugin/AuditEvent.java | 7 +
.../java/org/apache/doris/qe/ConnectProcessor.java | 7 +-
.../org/apache/doris/planner/SqlDigestTest.java | 142 +++++++++++++++++++++
.../org/apache/doris/utframe/UtFrameUtils.java | 15 +++
.../doris/plugin/audit/AuditLoaderPlugin.java | 1 +
.../doris/plugin/audit/DorisStreamLoader.java | 4 +-
37 files changed, 735 insertions(+), 5 deletions(-)
diff --git a/docs/en/ecosystem/audit-plugin.md b/docs/en/ecosystem/audit-plugin.md
index 542ddae936..0c71aa73ed 100644
--- a/docs/en/ecosystem/audit-plugin.md
+++ b/docs/en/ecosystem/audit-plugin.md
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
frontend_ip varchar(32) comment "Frontend ip of executing this statement",
cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
sql_hash varchar(50) comment "Hash value for this query",
+ sql_digest varchar(48) comment "Sql digest for this query",
peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
stmt string comment "The original statement, trimed if longer than 2G"
) engine=OLAP
@@ -97,4 +98,4 @@ The `dynamic_partition` attribute selects the number of days to keep the audit l
After that, connect to Doris and use the `INSTALL PLUGIN` command to complete the installation. After successful installation, you can see the installed plug-ins through `SHOW PLUGINS`, and the status is `INSTALLED`.
-Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
\ No newline at end of file
+Upon completion, the plug-in will continuously import audit date into this table at specified intervals.
diff --git a/docs/zh-CN/ecosystem/audit-plugin.md b/docs/zh-CN/ecosystem/audit-plugin.md
index fbd203d4b9..71bd5cc25c 100644
--- a/docs/zh-CN/ecosystem/audit-plugin.md
+++ b/docs/zh-CN/ecosystem/audit-plugin.md
@@ -72,6 +72,7 @@ create table doris_audit_tbl__
frontend_ip varchar(32) comment "Frontend ip of executing this statement",
cpu_time_ms bigint comment "Total scan cpu time in millisecond of this query",
sql_hash varchar(48) comment "Hash value for this query",
+ sql_digest varchar(48) comment "Sql digest for this query",
peak_memory_bytes bigint comment "Peak memory bytes used on all backends of this query",
stmt string comment "The original statement, trimed if longer than 2G "
) engine=OLAP
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
index 7b1b208596..35a460df47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticExpr.java
@@ -838,6 +838,36 @@ public class AnalyticExpr extends Expr {
return sb.toString();
}
+ @Override
+ public String toDigestImpl() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(fnCall.toDigest()).append(" OVER (");
+ boolean needsSpace = false;
+ if (!partitionExprs.isEmpty()) {
+ sb.append("PARTITION BY ").append(exprListToDigest(partitionExprs));
+ needsSpace = true;
+ }
+ if (!orderByElements.isEmpty()) {
+ List<String> orderByStrings = Lists.newArrayList();
+ for (OrderByElement e : orderByElements) {
+ orderByStrings.add(e.toDigest());
+ }
+ if (needsSpace) {
+ sb.append(" ");
+ }
+ sb.append("ORDER BY ").append(Joiner.on(", ").join(orderByStrings));
+ needsSpace = true;
+ }
+ if (window != null) {
+ if (needsSpace) {
+ sb.append(" ");
+ }
+ sb.append(window.toDigest());
+ }
+ sb.append(")");
+ return sb.toString();
+ }
+
private String exprListToSql(List<? extends Expr> exprs) {
if (exprs == null || exprs.isEmpty())
return "";
@@ -847,4 +877,15 @@ public class AnalyticExpr extends Expr {
}
return Joiner.on(", ").join(strings);
}
+
+ private String exprListToDigest(List<? extends Expr> exprs) {
+ if (exprs == null || exprs.isEmpty()) {
+ return "";
+ }
+ List<String> strings = Lists.newArrayList();
+ for (Expr expr : exprs) {
+ strings.add(expr.toDigest());
+ }
+ return Joiner.on(", ").join(strings);
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java
index 65372f77bd..2acceb9275 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AnalyticWindow.java
@@ -167,6 +167,17 @@ public class AnalyticWindow {
return sb.toString();
}
+ public String toDigest() {
+ StringBuilder sb = new StringBuilder();
+
+ if (expr != null) {
+ sb.append(expr.toDigest()).append(" ");
+ }
+
+ sb.append(type.toString());
+ return sb.toString();
+ }
+
public TAnalyticWindowBoundary toThrift(Type windowType) {
TAnalyticWindowBoundary result = new TAnalyticWindowBoundary(type.toThrift());
@@ -296,6 +307,21 @@ public class AnalyticWindow {
return sb.toString();
}
+ public String toDigest() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(type_.toString()).append(" ");
+
+ if (rightBoundary_ == null) {
+ sb.append(leftBoundary_.toDigest());
+ } else {
+ sb.append("BETWEEN ").append(leftBoundary_.toDigest()).append(" AND ");
+ sb.append(rightBoundary_.toDigest());
+ }
+
+ return sb.toString();
+ }
+
+
public TAnalyticWindow toThrift() {
TAnalyticWindow result = new TAnalyticWindow(type_.toThrift());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
index 20b5528247..e614881603 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
@@ -221,6 +221,15 @@ public class ArithmeticExpr extends Expr {
}
}
+ @Override
+ public String toDigestImpl() {
+ if (children.size() == 1) {
+ return op.toString() + " " + getChild(0).toDigest();
+ } else {
+ return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
+ }
+ }
+
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.ARITHMETIC_EXPR;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
index 8a9ed1dabf..7e06d1b7aa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArrayLiteral.java
@@ -72,6 +72,14 @@ public class ArrayLiteral extends LiteralExpr {
return "ARRAY(" + StringUtils.join(list, ", ") + ")";
}
+ @Override
+ public String toDigestImpl() {
+ List<String> list = new ArrayList<>(children.size());
+ children.forEach(v -> list.add(v.toDigestImpl()));
+
+ return "ARRAY(" + StringUtils.join(list, ", ") + ")";
+ }
+
@Override
public String getStringValue() {
List<String> list = new ArrayList<>(children.size());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java
index f3c836d72f..f1f00de50a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BetweenPredicate.java
@@ -96,7 +96,14 @@ public class BetweenPredicate extends Predicate {
public String toSqlImpl() {
String notStr = (isNotBetween) ? "NOT " : "";
return children.get(0).toSql() + " " + notStr + "BETWEEN " +
- children.get(1).toSql() + " AND " + children.get(2).toSql();
+ children.get(1).toSql() + " AND " + children.get(2).toSql();
+ }
+
+ @Override
+ public String toDigestImpl() {
+ String notStr = (isNotBetween) ? "NOT " : "";
+ return children.get(0).toDigest() + " " + notStr + "BETWEEN " +
+ children.get(1).toDigest() + " AND " + children.get(2).toDigest();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
index a37705293a..c23659edf5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BinaryPredicate.java
@@ -242,6 +242,11 @@ public class BinaryPredicate extends Predicate implements Writable {
return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql();
}
+ @Override
+ public String toDigestImpl() {
+ return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
+ }
+
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.BINARY_PRED;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java
index 6e2851a89d..8a41e91d35 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CaseExpr.java
@@ -129,6 +129,24 @@ public class CaseExpr extends Expr {
return output.toString();
}
+ @Override
+ public String toDigestImpl() {
+ StringBuilder sb = new StringBuilder("CASE");
+ int childIdx = 0;
+ if (hasCaseExpr) {
+ sb.append(" ").append(children.get(childIdx++).toDigest());
+ }
+ while (childIdx + 2 <= children.size()) {
+ sb.append(" WHEN ").append(children.get(childIdx++).toDigest());
+ sb.append(" THEN ").append(children.get(childIdx++).toDigest());
+ }
+ if (hasElseExpr) {
+ sb.append(" ELSE ").append(children.get(children.size() - 1).toDigest());
+ }
+ sb.append(" END");
+ return sb.toString();
+ }
+
@Override
public boolean isVectorized() {
return false;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 5e024ae784..8c71a0e6ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -206,6 +206,27 @@ public class CastExpr extends Expr {
}
}
+ @Override
+ public String toDigestImpl() {
+ boolean isVerbose = ConnectContext.get() != null &&
+ ConnectContext.get().getExecutor() != null &&
+ ConnectContext.get().getExecutor().getParsedStmt() != null &&
+ ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions() != null &&
+ ConnectContext.get().getExecutor().getParsedStmt().getExplainOptions().isVerbose();
+ if (isImplicit && !isVerbose) {
+ return getChild(0).toDigest();
+ }
+ if (isAnalyzed) {
+ if (type.isStringType()) {
+ return "CAST(" + getChild(0).toDigest() + " AS " + "CHARACTER" + ")";
+ } else {
+ return "CAST(" + getChild(0).toDigest() + " AS " + type.toString() + ")";
+ }
+ } else {
+ return "CAST(" + getChild(0).toDigest() + " AS " + targetTypeDef.toString() + ")";
+ }
+ }
+
@Override
protected void treeToThriftHelper(TExpr container) {
if (noOp) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
index 64afaee10c..d1e94e03e2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CompoundPredicate.java
@@ -94,6 +94,15 @@ public class CompoundPredicate extends Predicate {
}
}
+ @Override
+ public String toDigestImpl() {
+ if (children.size() == 1) {
+ return "NOT " + getChild(0).toDigest();
+ } else {
+ return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
+ }
+ }
+
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.COMPOUND_PRED;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java
index 615cbc7f13..7a4d9512ff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ExistsPredicate.java
@@ -58,6 +58,7 @@ public class ExistsPredicate extends Predicate {
@Override
public Expr clone() { return new ExistsPredicate(this); }
+ @Override
public String toSqlImpl() {
StringBuilder strBuilder = new StringBuilder();
if (notExists) {
@@ -69,6 +70,18 @@ public class ExistsPredicate extends Predicate {
return strBuilder.toString();
}
+ @Override
+ public String toDigestImpl() {
+ StringBuilder strBuilder = new StringBuilder();
+ if (notExists) {
+ strBuilder.append("NOT ");
+
+ }
+ strBuilder.append("EXISTS ");
+ strBuilder.append(getChild(0).toDigest());
+ return strBuilder.toString();
+ }
+
@Override
public int hashCode() {
return 31 * super.hashCode() + Boolean.hashCode(notExists);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index 4ce1224a13..e53d6444eb 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -874,12 +874,25 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
return (printSqlInParens) ? "(" + toSqlImpl() + ")" : toSqlImpl();
}
+ public String toDigest() {
+ return (printSqlInParens) ? "(" + toDigestImpl() + ")" : toDigestImpl();
+ }
+
/**
* Returns a SQL string representing this expr. Subclasses should override this method
* instead of toSql() to ensure that parenthesis are properly added around the toSql().
*/
protected abstract String toSqlImpl();
+ /**
+ * !!!!!! Important !!!!!!
+ * Subclasses should override this method if
+ * sql digest should be represented different from tosqlImpl().
+ */
+ protected String toDigestImpl() {
+ return toSqlImpl();
+ }
+
public String toMySql() {
return toSql();
}
@@ -952,6 +965,14 @@ abstract public class Expr extends TreeNode<Expr> implements ParseNode, Cloneabl
// }
// }
+ public List<String> childrenToDigest() {
+ List<String> childrenDigestList = Lists.newArrayList();
+ for (Expr child : children) {
+ childrenDigestList.add(child.toDigest());
+ }
+ return childrenDigestList;
+ }
+
public static com.google.common.base.Predicate<Expr> isAggregatePredicate() {
return IS_AGGREGATE_PREDICATE;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
index b2f5e29f87..8c1ab2c0ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
@@ -193,6 +193,17 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
return builder.toString();
}
+ public String toDigest() {
+ StringBuilder builder = new StringBuilder();
+ if (!tableRefs_.isEmpty()) {
+ builder.append(" FROM");
+ for (int i = 0; i < tableRefs_.size(); ++i) {
+ builder.append(" " + tableRefs_.get(i).toDigest());
+ }
+ }
+ return builder.toString();
+ }
+
public boolean isEmpty() { return tableRefs_.isEmpty(); }
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index cb03fc0039..305bdc47dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -292,6 +292,61 @@ public class FunctionCallExpr extends Expr {
return sb.toString();
}
+ private String paramsToDigest() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("(");
+
+ if (fnParams.isStar()) {
+ sb.append("*");
+ }
+ if (fnParams.isDistinct()) {
+ sb.append("DISTINCT ");
+ }
+ int len = children.size();
+ List<String> result = Lists.newArrayList();
+ if (fnName.getFunction().equalsIgnoreCase("json_array") ||
+ fnName.getFunction().equalsIgnoreCase("json_object")) {
+ len = len - 1;
+ }
+ if (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
+ fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
+ fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
+ fnName.getFunction().equalsIgnoreCase("sm4_encrypt")) {
+ len = len - 1;
+ }
+ for (int i = 0; i < len; ++i) {
+ if (i == 1 && (fnName.getFunction().equalsIgnoreCase("aes_decrypt") ||
+ fnName.getFunction().equalsIgnoreCase("aes_encrypt") ||
+ fnName.getFunction().equalsIgnoreCase("sm4_decrypt") ||
+ fnName.getFunction().equalsIgnoreCase("sm4_encrypt"))) {
+ result.add("\'***\'");
+ } else {
+ result.add(children.get(i).toDigest());
+ }
+ }
+ sb.append(Joiner.on(", ").join(result)).append(")");
+ return sb.toString();
+ }
+
+ @Override
+ public String toDigestImpl() {
+ Expr expr;
+ if (originStmtFnExpr != null) {
+ expr = originStmtFnExpr;
+ } else {
+ expr = this;
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append(((FunctionCallExpr) expr).fnName);
+ sb.append(paramsToDigest());
+ if (fnName.getFunction().equalsIgnoreCase("json_quote") ||
+ fnName.getFunction().equalsIgnoreCase("json_array") ||
+ fnName.getFunction().equalsIgnoreCase("json_object")) {
+ return forJSON(sb.toString());
+ }
+ return sb.toString();
+ }
+
@Override
public String debugString() {
return MoreObjects.toStringHelper(this)/*.add("op", aggOp)*/.add("name", fnName).add("isStar",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
index e30d8fde19..1bcac67a78 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InPredicate.java
@@ -269,6 +269,19 @@ public class InPredicate extends Predicate {
return strBuilder.toString();
}
+ @Override
+ public String toDigestImpl() {
+ StringBuilder strBuilder = new StringBuilder();
+ String notStr = (isNotIn) ? "NOT " : "";
+ strBuilder.append(getChild(0).toDigest() + " " + notStr + "IN (");
+ for (int i = 1; i < children.size(); ++i) {
+ strBuilder.append(getChild(i).toDigest());
+ strBuilder.append((i + 1 != children.size()) ? ", " : "");
+ }
+ strBuilder.append(")");
+ return strBuilder.toString();
+ }
+
@Override
public String toString() {
return toSql();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
index 7700ef8507..a708cd60e8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
@@ -463,4 +463,24 @@ public class InlineViewRef extends TableRef {
return sb.toString();
}
+
+ @Override
+ public String tableRefToDigest() {
+ String aliasSql = null;
+ String alias = getExplicitAlias();
+ if (alias != null) {
+ aliasSql = ToSqlUtils.getIdentSql(alias);
+ }
+ if (view != null) {
+ return name.toSql() + (aliasSql == null ? "" : " " + aliasSql);
+ }
+
+ StringBuilder sb = new StringBuilder()
+ .append("(")
+ .append(queryStmt.toDigest())
+ .append(") ")
+ .append(aliasSql);
+
+ return sb.toString();
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
index c266e93bae..d63dbbff57 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/IsNullPredicate.java
@@ -97,6 +97,11 @@ public class IsNullPredicate extends Predicate {
return getChild(0).toSql() + (isNotNull ? " IS NOT NULL" : " IS NULL");
}
+ @Override
+ public String toDigestImpl() {
+ return getChild(0).toDigest() + (isNotNull ? " IS NOT NULL" : " IS NULL");
+ }
+
public boolean isSlotRefChildren() {
return (children.get(0) instanceof SlotRef);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
index 9cbf924bf4..05cee02937 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LikePredicate.java
@@ -108,6 +108,12 @@ public class LikePredicate extends Predicate {
return getChild(0).toSql() + " " + op.toString() + " " + getChild(1).toSql();
}
+ @Override
+ public String toDigestImpl() {
+ return getChild(0).toDigest() + " " + op.toString() + " " + getChild(1).toDigest();
+ }
+
+
@Override
protected void toThrift(TExprNode msg) {
msg.node_type = TExprNodeType.FUNCTION_CALL;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LimitElement.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LimitElement.java
index 80e3b2a46b..7d5f06ac7e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LimitElement.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LimitElement.java
@@ -96,6 +96,18 @@ public class LimitElement {
return sb.toString();
}
+ public String toDigest() {
+ if (limit == -1) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder(" LIMIT ");
+ if (offset != 0) {
+ sb.append(offset + "?, ");
+ }
+ sb.append("" + " ? ");
+ return sb.toString();
+ }
+
public void analyze(Analyzer analyzer) {
if (limit == 0) analyzer.setHasEmptyResultSet();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
index 18c5b0d132..9ae73092cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LiteralExpr.java
@@ -198,6 +198,11 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
return buffer;
}
+ @Override
+ public String toDigestImpl() {
+ return " ? ";
+ }
+
// Swaps the sign of numeric literals.
// Throws for non-numeric literals.
public void swapSign() throws NotImplementedException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java
index 24ee9f5f59..aadb13b171 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OrderByElement.java
@@ -133,6 +133,22 @@ public class OrderByElement {
return strBuilder.toString();
}
+ public String toDigest() {
+ StringBuilder strBuilder = new StringBuilder();
+ strBuilder.append(expr.toDigest());
+ strBuilder.append(isAsc ? " ASC" : " DESC");
+ if (nullsFirstParam != null) {
+ if (isAsc && nullsFirstParam) {
+ // If ascending, nulls are last by default, so only add if nulls first.
+ strBuilder.append(" NULLS FIRST");
+ } else if (!isAsc && !nullsFirstParam) {
+ // If descending, nulls are first by default, so only add if nulls last.
+ strBuilder.append(" NULLS LAST");
+ }
+ }
+ return strBuilder.toString();
+ }
+
@Override
public String toString() {
return toSql();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index 3dd04cc800..8df9a0477b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -529,6 +529,15 @@ public class OutFileClause {
return sb.toString();
}
+ public String toDigest() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(" INTO OUTFILE '").append(" ? ").append(" FORMAT AS ").append(" ? ");
+ if (properties != null && !properties.isEmpty()) {
+ sb.append(" PROPERTIES(").append(" ? ").append(")");
+ }
+ return sb.toString();
+ }
+
public TResultFileSinkOptions toSinkOptions() {
TResultFileSinkOptions sinkOptions = new TResultFileSinkOptions(filePath, fileFormatType);
if (isCsvFormat()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
index 47e9c244ea..8ddb4f024b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/QueryStmt.java
@@ -706,6 +706,10 @@ public abstract class QueryStmt extends StatementBase {
return outFileClause != null ? outFileClause.clone() : null;
}
+ public String toDigest() {
+ return "";
+ }
+
/**
* C'tor for cloning.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java
index 7ad88a24b9..2c5bd245f5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectListItem.java
@@ -97,6 +97,20 @@ public class SelectListItem {
}
}
+ public String toDigest() {
+ if (!isStar) {
+ Preconditions.checkNotNull(expr);
+ String aliasSql = null;
+ if (alias != null) {
+ aliasSql = "`" + alias + "`";
+ }
+ return expr.toDigest() + ((aliasSql == null) ? "" : " " + aliasSql);
+ } else if (tblName != null) {
+ return tblName.toString() + ".*";
+ } else {
+ return "*";
+ }
+ }
/**
* Return a column label for the select list item.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 9bf0875c12..f85b618c55 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -1710,6 +1710,83 @@ public class SelectStmt extends QueryStmt {
return strBuilder.toString();
}
+ @Override
+ public String toDigest() {
+ StringBuilder strBuilder = new StringBuilder();
+ if (withClause_ != null) {
+ strBuilder.append(withClause_.toDigest());
+ strBuilder.append(" ");
+ }
+
+ // Select list
+ strBuilder.append("SELECT ");
+ if (selectList.isDistinct()) {
+ strBuilder.append("DISTINCT ");
+ }
+
+ if (originalExpr == null) {
+ originalExpr = Expr.cloneList(resultExprs);
+ }
+
+ if (resultExprs.isEmpty()) {
+ for (int i = 0; i < selectList.getItems().size(); ++i) {
+ if (i != 0) {
+ strBuilder.append(", ");
+ }
+ strBuilder.append(selectList.getItems().get(i).toDigest());
+ }
+ } else {
+ for (int i = 0; i < originalExpr.size(); ++i) {
+ if (i != 0) {
+ strBuilder.append(", ");
+ }
+ strBuilder.append(originalExpr.get(i).toDigest());
+ strBuilder.append(" AS ").append(SqlUtils.getIdentSql(colLabels.get(i)));
+ }
+ }
+
+ // From clause
+ if (!fromClause_.isEmpty()) {
+ strBuilder.append(fromClause_.toDigest());
+ }
+
+ // Where clause
+ if (whereClause != null) {
+ strBuilder.append(" WHERE ");
+ strBuilder.append(whereClause.toDigest());
+ }
+ // Group By clause
+ if (groupByClause != null) {
+ strBuilder.append(" GROUP BY ");
+ strBuilder.append(groupByClause.toSql());
+ }
+ // Having clause
+ if (havingClause != null) {
+ strBuilder.append(" HAVING ");
+ strBuilder.append(havingClause.toDigest());
+ }
+ // Order By clause
+ if (orderByElements != null) {
+ strBuilder.append(" ORDER BY ");
+ for (int i = 0; i < orderByElements.size(); ++i) {
+ strBuilder.append(orderByElements.get(i).getExpr().toDigest());
+ if (sortInfo != null) {
+ strBuilder.append((sortInfo.getIsAscOrder().get(i)) ? " ASC" : " DESC");
+ }
+ strBuilder.append((i + 1 != orderByElements.size()) ? ", " : "");
+ }
+ }
+ // Limit clause.
+ if (hasLimitClause()) {
+ strBuilder.append(limitElement.toDigest());
+ }
+
+ if (hasOutFileClause()) {
+ strBuilder.append(outFileClause.toDigest());
+ }
+ return strBuilder.toString();
+ }
+
/**
* If the select statement has a sort/top that is evaluated, then the sort tuple
* is materialized. Else, if there is aggregation then the aggregate tuple id is
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
index acdd28e595..0162dcad13 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SetOperationStmt.java
@@ -659,6 +659,57 @@ public class SetOperationStmt extends QueryStmt {
return strBuilder.toString();
}
+ @Override
+ public String toDigest() {
+ StringBuilder strBuilder = new StringBuilder();
+ if (withClause_ != null) {
+ strBuilder.append(withClause_.toDigest());
+ strBuilder.append(" ");
+ }
+
+ strBuilder.append(operands.get(0).getQueryStmt().toDigest());
+ for (int i = 1; i < operands.size() - 1; ++i) {
+ strBuilder.append(
+ " " + operands.get(i).getOperation().toString() + " "
+ + ((operands.get(i).getQualifier() == Qualifier.ALL) ? "ALL " : ""));
+ if (operands.get(i).getQueryStmt() instanceof SetOperationStmt) {
+ strBuilder.append("(");
+ }
+ strBuilder.append(operands.get(i).getQueryStmt().toDigest());
+ if (operands.get(i).getQueryStmt() instanceof SetOperationStmt) {
+ strBuilder.append(")");
+ }
+ }
+ // Determine whether we need parenthesis around the last Set operand.
+ SetOperand lastOperand = operands.get(operands.size() - 1);
+ QueryStmt lastQueryStmt = lastOperand.getQueryStmt();
+ strBuilder.append(" " + lastOperand.getOperation().toString() + " "
+ + ((lastOperand.getQualifier() == Qualifier.ALL) ? "ALL " : ""));
+ if (lastQueryStmt instanceof SetOperationStmt || ((hasOrderByClause() || hasLimitClause()) &&
+ !lastQueryStmt.hasLimitClause() &&
+ !lastQueryStmt.hasOrderByClause())) {
+ strBuilder.append("(");
+ strBuilder.append(lastQueryStmt.toDigest());
+ strBuilder.append(")");
+ } else {
+ strBuilder.append(lastQueryStmt.toDigest());
+ }
+ // Order By clause
+ if (hasOrderByClause()) {
+ strBuilder.append(" ORDER BY ");
+ for (int i = 0; i < orderByElements.size(); ++i) {
+ strBuilder.append(orderByElements.get(i).getExpr().toDigest());
+ strBuilder.append(orderByElements.get(i).getIsAsc() ? " ASC" : " DESC");
+ strBuilder.append((i + 1 != orderByElements.size()) ? ", " : "");
+ }
+ }
+ // Limit clause.
+ if (hasLimitClause()) {
+ strBuilder.append(limitElement.toDigest());
+ }
+ return strBuilder.toString();
+ }
+
@Override
public ArrayList<String> getColLabels() {
Preconditions.checkState(operands.size() > 0);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java
index 1934236b87..4a0501bd6c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Subquery.java
@@ -56,6 +56,11 @@ public class Subquery extends Expr {
@Override
public String toSqlImpl() { return "(" + stmt.toSql() + ")"; }
+ @Override
+ public String toDigestImpl() {
+ return "(" + stmt.toDigest() + ")";
+ }
+
/**
* C'tor that initializes a Subquery from a QueryStmt.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
index 7536c370a0..2375a523ca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
@@ -649,6 +649,10 @@ public class TableRef implements ParseNode, Writable {
return tblName;
}
+ public String tableRefToDigest() {
+ return tableRefToSql();
+ }
+
@Override
public String toSql() {
if (joinOp == null) {
@@ -670,6 +674,26 @@ public class TableRef implements ParseNode, Writable {
return output.toString();
}
+ public String toDigest() {
+ if (joinOp == null) {
+ // prepend "," if we're part of a sequence of table refs w/o an
+ // explicit JOIN clause
+ return (leftTblRef != null ? ", " : "") + tableRefToDigest();
+ }
+
+ StringBuilder output = new StringBuilder(" " + joinOpToSql() + " ");
+ if (joinHints != null && !joinHints.isEmpty()) {
+ output.append("[").append(Joiner.on(", ").join(joinHints)).append("] ");
+ }
+ output.append(tableRefToDigest()).append(" ");
+ if (usingColNames != null) {
+ output.append("USING (").append(Joiner.on(", ").join(usingColNames)).append(")");
+ } else if (onClause != null) {
+ output.append("ON ").append(onClause.toDigest());
+ }
+ return output.toString();
+ }
+
public String getAlias() {
if (!hasExplicitAlias()) {
return name.toString();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
index c04f16c6b8..f398cc8340 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TimestampArithmeticExpr.java
@@ -325,6 +325,44 @@ public class TimestampArithmeticExpr extends Expr {
return strBuilder.toString();
}
+ @Override
+ public String toDigestImpl() {
+ StringBuilder strBuilder = new StringBuilder();
+ if (funcName != null) {
+ if (funcName.equalsIgnoreCase("TIMESTAMPDIFF") || funcName.equalsIgnoreCase("TIMESTAMPADD")) {
+ strBuilder.append(funcName).append("(");
+ strBuilder.append(timeUnitIdent).append(", ");
+ strBuilder.append(getChild(1).toDigest()).append(", ");
+ strBuilder.append(getChild(0).toDigest()).append(")");
+ return strBuilder.toString();
+ }
+ // Function-call like version.
+ strBuilder.append(funcName).append("(");
+ strBuilder.append(getChild(0).toDigest()).append(", ");
+ strBuilder.append("INTERVAL ");
+ strBuilder.append(getChild(1).toDigest());
+ strBuilder.append(" ").append(timeUnitIdent);
+ strBuilder.append(")");
+ return strBuilder.toString();
+ }
+ if (intervalFirst) {
+ // Non-function-call like version with interval as first operand.
+ strBuilder.append("INTERVAL ");
+ strBuilder.append(getChild(1).toDigest() + " ");
+ strBuilder.append(timeUnitIdent);
+ strBuilder.append(" ").append(op.toString()).append(" ");
+ strBuilder.append(getChild(0).toDigest());
+ } else {
+ // Non-function-call like version with interval as second operand.
+ strBuilder.append(getChild(0).toDigest());
+ strBuilder.append(" " + op.toString() + " ");
+ strBuilder.append("INTERVAL ");
+ strBuilder.append(getChild(1).toDigest() + " ");
+ strBuilder.append(timeUnitIdent);
+ }
+ return strBuilder.toString();
+ }
+
// Time units supported in timestamp arithmetic.
public enum TimeUnit {
YEAR("YEAR"), // YEARS
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java
index 73bfe0d58d..c6b143cfa7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/WithClause.java
@@ -143,5 +143,20 @@ public class WithClause implements ParseNode {
return "WITH " + Joiner.on(",").join(viewStrings);
}
+ public String toDigest() {
+ List<String> viewStrings = Lists.newArrayList();
+ for (View view : views_) {
+ // Enclose the view alias and explicit labels in quotes if Hive cannot parse it
+ // without quotes. This is needed for view compatibility between Impala and Hive.
+ String aliasSql = ToSqlUtils.getIdentSql(view.getName());
+ if (view.hasColLabels()) {
+ aliasSql += "(" + Joiner.on(", ").join(
+ ToSqlUtils.getIdentSqlList(view.getOriginalColLabels())) + ")";
+ }
+ viewStrings.add(aliasSql + " AS (" + view.getQueryStmt().toDigest() + ")");
+ }
+ return "WITH " + Joiner.on(",").join(viewStrings);
+ }
+
public List<View> getViews() { return views_; }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
index 500d6f80b9..4a2e9074fa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
@@ -84,6 +84,8 @@ public class AuditEvent {
public String sqlHash = "";
@AuditField(value = "peakMemoryBytes")
public long peakMemoryBytes = -1;
+ @AuditField(value = "SqlDigest")
+ public String sqlDigest = "";
public static class AuditEventBuilder {
@@ -186,6 +188,11 @@ public class AuditEvent {
return this;
}
+ public AuditEventBuilder setSqlDigest(String sqlDigest) {
+ auditEvent.sqlDigest = sqlDigest;
+ return this;
+ }
+
public AuditEvent build() {
return this.auditEvent;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index 65cded1472..f8507e73ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -19,6 +19,7 @@ package org.apache.doris.qe;
import org.apache.doris.analysis.InsertStmt;
import org.apache.doris.analysis.KillStmt;
+import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
@@ -29,6 +30,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Table;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
@@ -133,6 +135,10 @@ public class ConnectProcessor {
} else {
// ok query
MetricRepo.HISTO_QUERY_LATENCY.update(elapseMs);
+ if (elapseMs > Config.qe_slow_log_ms) {
+ String sqlDigest = DigestUtils.md5Hex(((QueryStmt) parsedStmt).toDigest());
+ ctx.getAuditEventBuilder().setSqlDigest(sqlDigest);
+ }
}
ctx.getAuditEventBuilder().setIsQuery(true);
ctx.getQueryDetail().setEventTime(endTime);
@@ -183,7 +189,6 @@ public class ConnectProcessor {
}
String sqlHash = DigestUtils.md5Hex(originStmt);
ctx.setSqlHash(sqlHash);
-
ctx.getAuditEventBuilder().reset();
ctx.getAuditEventBuilder()
.setTimestamp(System.currentTimeMillis())
diff --git a/fe/fe-core/src/test/java/org/apache/doris/planner/SqlDigestTest.java b/fe/fe-core/src/test/java/org/apache/doris/planner/SqlDigestTest.java
new file mode 100644
index 0000000000..3247a75550
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/SqlDigestTest.java
@@ -0,0 +1,142 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.CreateDbStmt;
+import org.apache.doris.analysis.CreateTableStmt;
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.utframe.UtFrameUtils;
+
+import org.junit.Assert;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.UUID;
+
+public class SqlDigestTest {
+
+ private static String runningDir = "fe/mocked/SqlDigestTest/" + UUID.randomUUID().toString() + "/";
+
+ private static ConnectContext connectContext;
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UtFrameUtils.createDorisCluster(runningDir);
+ connectContext = UtFrameUtils.createDefaultCtx();
+
+ String createDbStmtStr = "create database db1;";
+ CreateDbStmt createDbStmt = (CreateDbStmt) UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, connectContext);
+ Catalog.getCurrentCatalog().createDb(createDbStmt);
+ // 3. create table tbl1
+ String createTblStmtStr = "create table db1.tbl1(k1 varchar(32), k2 varchar(32), k3 varchar(32), k4 int) "
+ + "AGGREGATE KEY(k1, k2,k3,k4) distributed by hash(k1) buckets 3 properties('replication_num' = '1');";
+ CreateTableStmt createTableStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, connectContext);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
+ }
+
+ @AfterClass
+ public static void tearDown() {
+ File file = new File(runningDir);
+ file.delete();
+ }
+
+ @Test
+ public void testWhere() throws Exception {
+ String sql1 = "select k4 from tbl1 where k1 > 1";
+ String sql2 = "select k4 from tbl1 where k1 > 5";
+ String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+
+ sql1 = "select k4 from tbl1 where k1 like 'xxx' ";
+ sql2 = "select k4 from tbl1 where k1 like 'kkskkkkkkkkk' ";
+ digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+
+
+ sql1 = "select k4 from tb1 where k1 < 2 and k2 > -1 ";
+ sql2 = "select k4 from tb1 where k1 < 1000 and k2 > 100000 ";
+ digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+
+ sql1 = "select k4 from tb1 where k1 < 2 or k2 > -1 ";
+ sql2 = "select k4 from tb1 where k1 < 3 or k2 > 100000 ";
+ digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+
+ sql1 = "select k4 from tb1 where not k1 < 2 ";
+ sql2 = "select k4 from tb1 where not k1 < 3";
+ digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+
+ sql1 = "select k4 from tb1 where not k1 < 2 ";
+ sql2 = "select k4 from tb1 where not k1 > 3";
+ digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertNotEquals(digest1, digest2);
+ }
+
+ @Test
+ public void testLimit() throws Exception {
+ String sql1 = "select k4 from tb1 where k1 > 1 limit 1";
+ String sql2 = "select k4 from tb1 where k1 > 5 limit 20";
+ String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+
+ sql1 = "select k4 from tb1 where k1 > 1 order by k1 limit 1";
+ sql2 = "select k4 from tb1 where k1 > 5 order by k1 limit 20";
+ digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+ }
+
+ @Test
+ public void testFunction() throws Exception {
+ String sql1 = "select substr(k4, 1, 2) from tb1 where k1 > 1 limit 1";
+ String sql2 = "select substr(k4, 1, 5) from tb1 where k1 > 1 limit 1";
+ String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+ }
+
+ @Test
+ public void testArithmetic() throws Exception {
+ String sql1 = "select k1 + 1 from tb1";
+ String sql2 = "select k1 + 2 from tb1";
+ String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+ }
+
+ @Test
+ public void testCaseWhen() throws Exception {
+ String sql1 = "select k1+20, case k2 when k3 then 1 else 0 end from tbl1 where k4 is null";
+ String sql2 = "select k1+20, case k2 when k3 then 1000 else 9999999 end from tbl1 where k4 is null";
+ String digest1 = UtFrameUtils.getStmtDigest(connectContext, sql1);
+ String digest2 = UtFrameUtils.getStmtDigest(connectContext, sql2);
+ Assert.assertEquals(digest1, digest2);
+ }
+}
\ No newline at end of file
diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
index 020250b94d..229e2d519e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
@@ -19,6 +19,7 @@ package org.apache.doris.utframe;
import org.apache.doris.analysis.Analyzer;
import org.apache.doris.analysis.ExplainOptions;
+import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SqlParser;
import org.apache.doris.analysis.SqlScanner;
import org.apache.doris.analysis.StatementBase;
@@ -45,9 +46,12 @@ import org.apache.doris.utframe.MockedFrontend.EnvVarNotSetException;
import org.apache.doris.utframe.MockedFrontend.FeStartException;
import org.apache.doris.utframe.MockedFrontend.NotInitException;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
+
+import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.io.FileUtils;
import java.io.File;
@@ -297,5 +301,16 @@ public class UtFrameUtils {
return null;
}
}
+
+ public static String getStmtDigest(ConnectContext connectContext, String originStmt) throws Exception {
+ SqlScanner input =
+ new SqlScanner(new StringReader(originStmt), connectContext.getSessionVariable().getSqlMode());
+ SqlParser parser = new SqlParser(input);
+ StatementBase statementBase = SqlParserUtils.getFirstStmt(parser);
+ Preconditions.checkState(statementBase instanceof QueryStmt);
+ QueryStmt queryStmt = (QueryStmt) statementBase;
+ String digest = queryStmt.toDigest();
+ return DigestUtils.md5Hex(digest);
+ }
}
diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
index 6d2b7bc14f..38c7d00bbf 100755
--- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
+++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
@@ -159,6 +159,7 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
auditBuffer.append(event.feIp).append("\t");
auditBuffer.append(event.cpuTimeMs).append("\t");
auditBuffer.append(event.sqlHash).append("\t");
+ auditBuffer.append(event.sqlDigest).append("\t");
auditBuffer.append(event.peakMemoryBytes).append("\t");
// trim the query to avoid too long
// use `getBytes().length` to get real byte length
diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
index 2311e2b06e..573b65a467 100644
--- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
+++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
@@ -70,7 +70,7 @@ public class DorisStreamLoader {
conn.addRequestProperty("label", label);
conn.addRequestProperty("max_filter_ratio", "1.0");
conn.addRequestProperty("columns", "query_id, time, client_ip, user, db, state, query_time, scan_bytes," +
- " scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, peak_memory_bytes, stmt");
+ " scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, sql_digest, peak_memory_bytes, stmt");
conn.setDoOutput(true);
conn.setDoInput(true);
@@ -87,7 +87,7 @@ public class DorisStreamLoader {
sb.append("-H \"").append("max_filter_ratio\":").append("\"1.0\" \\\n ");
sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, db, state, query_time," +
" scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash," +
- " peak_memory_bytes, stmt\" \\\n ");
+ " sql_digest, peak_memory_bytes, stmt\" \\\n ");
sb.append("\"").append(conn.getURL()).append("\"");
return sb.toString();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org