You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/22 15:31:07 UTC
[doris] branch branch-1.2-lts updated (5e35e74682 -> 1f9a5e8c42)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 5e35e74682 [Enhancement](export) cancel all running coordinators when execute cancel-export statement. (#15801)
new 77f6af5a87 [Feature](audit) add errorCode and errorMessage in audit log (#14925)
new 1f9a5e8c42 [chore](fe) update fe snapshot to 1.2 and fix auditloader compile error (#15787)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
build.sh | 4 +++-
docs/en/docs/ecosystem/audit-plugin.md | 4 ++++
docs/zh-CN/docs/ecosystem/audit-plugin.md | 4 ++++
.../src/main/java/org/apache/doris/plugin/AuditEvent.java | 15 ++++++++++++++-
.../main/java/org/apache/doris/qe/ConnectProcessor.java | 6 +++++-
.../src/main/java/org/apache/doris/qe/QueryState.java | 1 +
fe/pom.xml | 2 +-
.../org/apache/doris/plugin/audit/AuditLoaderPlugin.java | 2 ++
.../org/apache/doris/plugin/audit/DorisStreamLoader.java | 14 ++++++--------
fe_plugins/pom.xml | 8 ++++++--
10 files changed, 46 insertions(+), 14 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/02: [chore](fe) update fe snapshot to 1.2 and fix auditloader compile error (#15787)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1f9a5e8c42fc313ba64b88850b8b8c226594beca
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Wed Jan 11 08:46:48 2023 +0800
[chore](fe) update fe snapshot to 1.2 and fix auditloader compile error (#15787)
This PR #14925 change some field of AuditEvent, so we need to upgrade the fe-core's SNAPSHOT to 1.2
because auditloader depends on fe-core
Already push the 1.2-SNAPSHOT to
https://repository.apache.org/content/repositories/snapshots/org/apache/doris/fe-core/1.2-SNAPSHOT/
---
build.sh | 4 +++-
fe/pom.xml | 2 +-
fe_plugins/pom.xml | 8 ++++++--
3 files changed, 10 insertions(+), 4 deletions(-)
diff --git a/build.sh b/build.sh
index 130018a18b..e4aea2a657 100755
--- a/build.sh
+++ b/build.sh
@@ -130,7 +130,7 @@ BUILD_BROKER=0
BUILD_AUDIT=0
BUILD_META_TOOL='OFF'
BUILD_SPARK_DPP=0
-BUILD_JAVA_UDF=1
+BUILD_JAVA_UDF=0
BUILD_HIVE_UDF=0
CLEAN=0
HELP=0
@@ -154,10 +154,12 @@ else
BUILD_FE=1
BUILD_SPARK_DPP=1
BUILD_HIVE_UDF=1
+ BUILD_JAVA_UDF=1
shift
;;
--be)
BUILD_BE=1
+ BUILD_JAVA_UDF=1
shift
;;
--broker)
diff --git a/fe/pom.xml b/fe/pom.xml
index 19c71ffdfd..993d0d3a07 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -243,7 +243,7 @@ under the License.
<mariadb-java-client.version>3.0.4</mariadb-java-client.version>
<dlf-metastore-client-hive2.version>0.2.14</dlf-metastore-client-hive2.version>
<hadoop.version>2.10.2</hadoop.version>
- <revision>1.0-SNAPSHOT</revision>
+ <revision>1.2-SNAPSHOT</revision>
<project.scm.id>github</project.scm.id>
</properties>
<profiles>
diff --git a/fe_plugins/pom.xml b/fe_plugins/pom.xml
index 38ed89eaee..bdb90d3f66 100644
--- a/fe_plugins/pom.xml
+++ b/fe_plugins/pom.xml
@@ -68,7 +68,7 @@ under the License.
</modules>
<properties>
<log4j2.version>2.18.0</log4j2.version>
- <doris.version>1.0-SNAPSHOT</doris.version>
+ <doris.version>1.2-SNAPSHOT</doris.version>
<project.scm.id>github</project.scm.id>
</properties>
<profiles>
@@ -101,7 +101,11 @@ under the License.
</property>
</activation>
<repositories>
- <!-- for java-cup -->
+ <repository>
+ <id>snapshots</id>
+ <name>apache snapshots maven repo https</name>
+ <url>https://repository.apache.org/content/repositories/snapshots/</url>
+ </repository>
<repository>
<id>cloudera-public</id>
<url>https://repository.cloudera.com/artifactory/public/</url>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/02: [Feature](audit) add errorCode and errorMessage in audit log (#14925)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 77f6af5a877f58b0c452db5943c4074531a23d11
Author: wxy <du...@gmail.com>
AuthorDate: Mon Jan 9 08:47:57 2023 +0800
[Feature](audit) add errorCode and errorMessage in audit log (#14925)
* [feat] add errorCode and errorMessage in audit log.
* [Feature](audit) add errorCode and errorMessage in audit log
Co-authored-by: wangxiangyu@360shuke.com <wa...@360shuke.com>
---
docs/en/docs/ecosystem/audit-plugin.md | 4 ++++
docs/zh-CN/docs/ecosystem/audit-plugin.md | 4 ++++
.../src/main/java/org/apache/doris/plugin/AuditEvent.java | 15 ++++++++++++++-
.../main/java/org/apache/doris/qe/ConnectProcessor.java | 6 +++++-
.../src/main/java/org/apache/doris/qe/QueryState.java | 1 +
.../org/apache/doris/plugin/audit/AuditLoaderPlugin.java | 2 ++
.../org/apache/doris/plugin/audit/DorisStreamLoader.java | 14 ++++++--------
7 files changed, 36 insertions(+), 10 deletions(-)
diff --git a/docs/en/docs/ecosystem/audit-plugin.md b/docs/en/docs/ecosystem/audit-plugin.md
index 2e533d07c7..e2a2b6cf17 100644
--- a/docs/en/docs/ecosystem/audit-plugin.md
+++ b/docs/en/docs/ecosystem/audit-plugin.md
@@ -65,6 +65,8 @@ create table doris_audit_db__.doris_audit_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
+ error_code int comment "Error code of failing query.",
+ error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",
@@ -99,6 +101,8 @@ create table doris_audit_db__.doris_slow_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
+ error_code int comment "Error code of failing query.",
+ error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",
diff --git a/docs/zh-CN/docs/ecosystem/audit-plugin.md b/docs/zh-CN/docs/ecosystem/audit-plugin.md
index aa1d752428..e4beee04d8 100644
--- a/docs/zh-CN/docs/ecosystem/audit-plugin.md
+++ b/docs/zh-CN/docs/ecosystem/audit-plugin.md
@@ -65,6 +65,8 @@ create table doris_audit_db__.doris_audit_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
+ error_code int comment "Error code of failing query.",
+ error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",
@@ -99,6 +101,8 @@ create table doris_audit_db__.doris_slow_log_tbl__
user varchar(64) comment "User name",
db varchar(96) comment "Database of this query",
state varchar(8) comment "Query result state. EOF, ERR, OK",
+ error_code int comment "Error code of failing query.",
+ error_message string comment "Error message of failing query.",
query_time bigint comment "Query execution time in millisecond",
scan_bytes bigint comment "Total scan bytes of this query",
scan_rows bigint comment "Total scan rows of this query",
diff --git a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
index 63a7d9fcb3..864a3e8029 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java
@@ -60,6 +60,10 @@ public class AuditEvent {
public String db = "";
@AuditField(value = "State")
public String state = "";
+ @AuditField(value = "ErrorCode")
+ public int errorCode = 0;
+ @AuditField(value = "ErrorMessage")
+ public String errorMessage = "";
@AuditField(value = "Time")
public long queryTime = -1;
@AuditField(value = "ScanBytes")
@@ -86,7 +90,6 @@ public class AuditEvent {
public long peakMemoryBytes = -1;
@AuditField(value = "SqlDigest")
public String sqlDigest = "";
-
@AuditField(value = "TraceId")
public String traceId = "";
@@ -131,6 +134,16 @@ public class AuditEvent {
return this;
}
+ public AuditEventBuilder setErrorCode(int errorCode) {
+ auditEvent.errorCode = errorCode;
+ return this;
+ }
+
+ public AuditEventBuilder setErrorMessage(String errorMessage) {
+ auditEvent.errorMessage = errorMessage;
+ return this;
+ }
+
public AuditEventBuilder setQueryTime(long queryTime) {
auditEvent.queryTime = queryTime;
return this;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
index fd3c978dd4..493126af54 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java
@@ -178,7 +178,11 @@ public class ConnectProcessor {
ctx.getAuditEventBuilder().setEventType(EventType.AFTER_QUERY)
.setDb(ClusterNamespace.getNameFromFullName(ctx.getDatabase()))
- .setState(ctx.getState().toString()).setQueryTime(elapseMs)
+ .setState(ctx.getState().toString())
+ .setErrorCode(ctx.getState().getErrorCode() == null ? 0 : ctx.getState().getErrorCode().getCode())
+ .setErrorMessage((ctx.getState().getErrorMessage() == null ? "" :
+ ctx.getState().getErrorMessage().replace("\n", " ").replace("\t", " ")))
+ .setQueryTime(elapseMs)
.setScanBytes(statistics == null ? 0 : statistics.getScanBytes())
.setScanRows(statistics == null ? 0 : statistics.getScanRows())
.setCpuTimeMs(statistics == null ? 0 : statistics.getCpuMs())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/QueryState.java b/fe/fe-core/src/main/java/org/apache/doris/qe/QueryState.java
index fd24d9a1ad..3cf2f22779 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/QueryState.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/QueryState.java
@@ -56,6 +56,7 @@ public class QueryState {
stateType = MysqlStateType.OK;
errorCode = null;
infoMessage = null;
+ errorMessage = "";
serverStatus = 0;
isQuery = false;
affectedRows = 0;
diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
index 992b35f686..e80383d19c 100755
--- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
+++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java
@@ -163,6 +163,8 @@ public class AuditLoaderPlugin extends Plugin implements AuditPlugin {
logBuffer.append(event.user).append("\t");
logBuffer.append(event.db).append("\t");
logBuffer.append(event.state).append("\t");
+ logBuffer.append(event.errorCode).append("\t");
+ logBuffer.append(event.errorMessage).append("\t");
logBuffer.append(event.queryTime).append("\t");
logBuffer.append(event.scanBytes).append("\t");
logBuffer.append(event.scanRows).append("\t");
diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
index 844ca04892..d2249a3ea7 100644
--- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
+++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/DorisStreamLoader.java
@@ -23,15 +23,12 @@ import org.apache.logging.log4j.Logger;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.IOException;
-import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Calendar;
-import java.util.List;
-import java.util.Map;
public class DorisStreamLoader {
private final static Logger LOG = LogManager.getLogger(DorisStreamLoader.class);
@@ -73,8 +70,9 @@ public class DorisStreamLoader {
conn.addRequestProperty("label", label);
conn.addRequestProperty("max_filter_ratio", "1.0");
- conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, db, state, query_time, scan_bytes," +
- " scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, sql_digest, peak_memory_bytes, stmt");
+ conn.addRequestProperty("columns", "query_id, `time`, client_ip, user, db, state, error_code, error_message, " +
+ "query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash, " +
+ "sql_digest, peak_memory_bytes, stmt");
conn.setDoOutput(true);
conn.setDoInput(true);
@@ -89,9 +87,9 @@ public class DorisStreamLoader {
sb.append("-H \"").append("Expect\":").append("\"100-continue\" \\\n ");
sb.append("-H \"").append("Content-Type\":").append("\"text/plain; charset=UTF-8\" \\\n ");
sb.append("-H \"").append("max_filter_ratio\":").append("\"1.0\" \\\n ");
- sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, db, state, query_time," +
- " scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, cpu_time_ms, sql_hash," +
- " sql_digest, peak_memory_bytes, stmt\" \\\n ");
+ sb.append("-H \"").append("columns\":").append("\"query_id, time, client_ip, user, db, state, error_code, " +
+ "error_message, query_time, scan_bytes, scan_rows, return_rows, stmt_id, is_query, frontend_ip, " +
+ "cpu_time_ms, sql_hash, sql_digest, peak_memory_bytes, stmt\" \\\n ");
sb.append("\"").append(conn.getURL()).append("\"");
return sb.toString();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org