You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/03 15:10:13 UTC
[doris] branch branch-1.2-lts updated (4c1c1ab505 -> bbf57a9dd0)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from 4c1c1ab505 [fix](like) fix wrong result of like pattern with backslash (#18351)
new b768b83583 [log](load) print detailed error message when publish failed (#18176)
new c3485b9fc3 [fix](memory) Remove page cache regular clear, disabled jemalloc prof by default (#18218)
new 51741e886e [fix](planner )need add LateralViewRef's id into TableRef's allTableRefIds (#18220)
new a59478a141 [Enhancement](spark load)Support spark time out config (#17108)
new ab45f53fb4 [fix](planner) Fix agg on inlineview which with constant slot (#18201)
new 1b2096502b [Fix](multi-catalog) fix hms automatic update error. (#18252)
new bbf57a9dd0 [test](jdbc catalog) add clickhouse jdbc catalog base type test (#18007)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/common/config.h | 4 -
be/src/common/daemon.cpp | 12 ---
be/src/olap/task/engine_publish_version_task.cpp | 3 +-
be/src/runtime/memory/mem_tracker_limiter.cpp | 5 +-
bin/start_be.sh | 4 +-
.../clickhouse.env} | 2 +-
.../clickhouse.yaml.tpl} | 29 ++++---
.../{mysql => clickhouse}/init/01-drop-db.sql | 2 +-
.../{mysql => clickhouse}/init/02-create-db.sql | 2 +-
.../clickhouse/init/03-create-table.sql | 96 +++++++++++++++++++++
.../docker-compose/clickhouse/init/04-insert.sql | 26 ++++++
docker/thirdparties/run-thirdparties-docker.sh | 21 ++++-
docs/en/community/developer-guide/be-vscode-dev.md | 2 +
docs/en/community/developer-guide/debug-tool.md | 31 ++++---
.../community/developer-guide/be-vscode-dev.md | 2 +
docs/zh-CN/community/developer-guide/debug-tool.md | 2 +-
.../java/org/apache/doris/analysis/SelectStmt.java | 14 ++-
.../java/org/apache/doris/analysis/TableRef.java | 6 ++
.../doris/datasource/HMSExternalCatalog.java | 4 +-
.../doris/load/loadv2/SparkEtlJobHandler.java | 5 +-
.../doris/load/loadv2/SparkLauncherMonitor.java | 22 ++++-
.../org/apache/doris/planner/AnalyticPlanner.java | 2 +-
.../apache/doris/planner/TableFunctionNode.java | 2 +-
.../load/loadv2/SparkLauncherMonitorTest.java | 7 +-
.../java/org/apache/doris/udf/JdbcExecutor.java | 1 +
regression-test/conf/regression-conf.groovy | 1 +
.../test_clickhouse_jdbc_catalog.out | Bin 0 -> 880 bytes
.../agg_on_view.out} | 0
.../test_inlineview_with_window_function.groovy} | 61 +++++++++----
....groovy => test_clickhouse_jdbc_catalog.groovy} | 34 +++-----
.../agg_on_view.groovy} | 35 ++++----
.../suites/trino_p0/constant_group_key.groovy | 2 +-
32 files changed, 314 insertions(+), 125 deletions(-)
copy docker/thirdparties/docker-compose/{postgresql/postgresql-14.env => clickhouse/clickhouse.env} (95%)
copy docker/thirdparties/docker-compose/{mysql/mysql-5.7.yaml.tpl => clickhouse/clickhouse.yaml.tpl} (69%)
copy docker/thirdparties/docker-compose/{mysql => clickhouse}/init/01-drop-db.sql (95%)
copy docker/thirdparties/docker-compose/{mysql => clickhouse}/init/02-create-db.sql (95%)
create mode 100644 docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
create mode 100644 docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql
create mode 100644 regression-test/data/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.out
copy regression-test/data/query_p0/{lateral_view/test_with_view.out => aggregate/agg_on_view.out} (100%)
copy regression-test/suites/{query_p1/return_binaray/test_return_binary_bitmap.groovy => correctness_p0/test_inlineview_with_window_function.groovy} (50%)
copy regression-test/suites/jdbc_catalog_p0/{test_sqlserver_jdbc_catalog.groovy => test_clickhouse_jdbc_catalog.groovy} (65%)
copy regression-test/suites/query_p0/{lateral_view/test_with_view.groovy => aggregate/agg_on_view.groovy} (62%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/07: [fix](planner) Fix agg on inlineview which with constant slot (#18201)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ab45f53fb4a74970d5b1403c4ffa0449d30a6dee
Author: AKIRA <33...@users.noreply.github.com>
AuthorDate: Fri Mar 31 00:54:37 2023 +0900
[fix](planner) Fix agg on inlineview which with constant slot (#18201)
Since slot that reference to constant has been marked as constant expr either, just add condition check to make sure such slot wouldn't be eliminated as constant from group exprs
---
.../java/org/apache/doris/analysis/SelectStmt.java | 2 +-
.../data/query_p0/aggregate/agg_on_view.out | 3 ++
.../suites/query_p0/aggregate/agg_on_view.groovy | 40 ++++++++++++++++++++++
.../suites/trino_p0/constant_group_key.groovy | 2 +-
4 files changed, 45 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 0baffbc15f..3c8e288fb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -1203,7 +1203,7 @@ public class SelectStmt extends QueryStmt {
for (Expr groupExpr : groupingExprs) {
//remove groupExpr if it is const, and it is not in select list
boolean removeConstGroupingKey = false;
- if (groupExpr.isConstant()) {
+ if (groupExpr.isConstant() && !(groupExpr.contains(e -> e instanceof SlotRef))) {
if (theFirstConstantGroupingExpr == null) {
theFirstConstantGroupingExpr = groupExpr;
}
diff --git a/regression-test/data/query_p0/aggregate/agg_on_view.out b/regression-test/data/query_p0/aggregate/agg_on_view.out
new file mode 100644
index 0000000000..9c9c4c6c8a
--- /dev/null
+++ b/regression-test/data/query_p0/aggregate/agg_on_view.out
@@ -0,0 +1,3 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !sql --
+
diff --git a/regression-test/suites/query_p0/aggregate/agg_on_view.groovy b/regression-test/suites/query_p0/aggregate/agg_on_view.groovy
new file mode 100644
index 0000000000..9f700283e1
--- /dev/null
+++ b/regression-test/suites/query_p0/aggregate/agg_on_view.groovy
@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("agg_on_view") {
+ sql """
+ create table test (
+ id int,
+ user_id int,
+ name varchar(20)
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ qt_sql """
+ select user_id,null_or_empty(tag) = 1,count(*)
+ from (
+ select *,
+ "abc" as tag
+ from test limit 10)t
+ group by user_id,tag
+ """
+}
\ No newline at end of file
diff --git a/regression-test/suites/trino_p0/constant_group_key.groovy b/regression-test/suites/trino_p0/constant_group_key.groovy
index cd159722a3..d666b5244c 100644
--- a/regression-test/suites/trino_p0/constant_group_key.groovy
+++ b/regression-test/suites/trino_p0/constant_group_key.groovy
@@ -36,7 +36,7 @@ suite("constant_group_key") {
explain {
sql("select a from (select '1' as b, 'abc' as a) T group by b, a")
- contains "group by: 'abc'"
+ contains "group by: '1', 'abc'"
}
sql "drop table if exists cgk_tbl"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/07: [test](jdbc catalog) add clickhouse jdbc catalog base type test (#18007)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit bbf57a9dd06ad98a045e722728ce249fc98d66e8
Author: yongkang.zhong <zh...@qq.com>
AuthorDate: Mon Apr 3 20:18:36 2023 +0800
[test](jdbc catalog) add clickhouse jdbc catalog base type test (#18007)
---
.../docker-compose/clickhouse/clickhouse.env | 19 ++++
.../docker-compose/clickhouse/clickhouse.yaml.tpl | 48 +++++++++++
.../docker-compose/clickhouse/init/01-drop-db.sql | 18 ++++
.../clickhouse/init/02-create-db.sql | 18 ++++
.../clickhouse/init/03-create-table.sql | 96 +++++++++++++++++++++
.../docker-compose/clickhouse/init/04-insert.sql | 26 ++++++
docker/thirdparties/run-thirdparties-docker.sh | 21 ++++-
.../java/org/apache/doris/udf/JdbcExecutor.java | 1 +
regression-test/conf/regression-conf.groovy | 1 +
.../test_clickhouse_jdbc_catalog.out | Bin 0 -> 880 bytes
.../test_clickhouse_jdbc_catalog.groovy | 64 ++++++++++++++
11 files changed, 309 insertions(+), 3 deletions(-)
diff --git a/docker/thirdparties/docker-compose/clickhouse/clickhouse.env b/docker/thirdparties/docker-compose/clickhouse/clickhouse.env
new file mode 100644
index 0000000000..13355650a6
--- /dev/null
+++ b/docker/thirdparties/docker-compose/clickhouse/clickhouse.env
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+DOCKER_CLICKHOUSE_EXTERNAL_HTTP_PORT=8123
diff --git a/docker/thirdparties/docker-compose/clickhouse/clickhouse.yaml.tpl b/docker/thirdparties/docker-compose/clickhouse/clickhouse.yaml.tpl
new file mode 100644
index 0000000000..b9e26d1c21
--- /dev/null
+++ b/docker/thirdparties/docker-compose/clickhouse/clickhouse.yaml.tpl
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+version: "2.1"
+
+services:
+ doris--clickhouse:
+ image: "clickhouse/clickhouse-server:22.8.15.23-alpine"
+ restart: always
+ environment:
+ CLICKHOUSE_PASSWORD: 123456
+ ulimits:
+ nofile:
+ soft: 262144
+ hard: 262144
+ ports:
+ - ${DOCKER_CLICKHOUSE_EXTERNAL_HTTP_PORT}:8123
+ healthcheck:
+ test: ["CMD-SHELL", "clickhouse-client --password=123456 --query 'SELECT 1'"]
+ interval: 30s
+ timeout: 10s
+ retries: 5
+ volumes:
+ - ./init:/docker-entrypoint-initdb.d
+ networks:
+ - doris--clickhouse
+ hello-world:
+ image: hello-world
+ depends_on:
+ doris--clickhouse:
+ condition: service_healthy
+
+networks:
+ doris--clickhouse:
diff --git a/docker/thirdparties/docker-compose/clickhouse/init/01-drop-db.sql b/docker/thirdparties/docker-compose/clickhouse/init/01-drop-db.sql
new file mode 100644
index 0000000000..469f104b98
--- /dev/null
+++ b/docker/thirdparties/docker-compose/clickhouse/init/01-drop-db.sql
@@ -0,0 +1,18 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+DROP DATABASE IF EXISTS doris_test;
diff --git a/docker/thirdparties/docker-compose/clickhouse/init/02-create-db.sql b/docker/thirdparties/docker-compose/clickhouse/init/02-create-db.sql
new file mode 100644
index 0000000000..da5a4cf993
--- /dev/null
+++ b/docker/thirdparties/docker-compose/clickhouse/init/02-create-db.sql
@@ -0,0 +1,18 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+CREATE DATABASE IF NOT EXISTS doris_test;
diff --git a/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
new file mode 100644
index 0000000000..4a8fceee3d
--- /dev/null
+++ b/docker/thirdparties/docker-compose/clickhouse/init/03-create-table.sql
@@ -0,0 +1,96 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+CREATE TABLE doris_test.type
+(
+ `k1` Bool,
+ `k2` Date,
+ `k3` Date32,
+ `k4` Datetime,
+ `k5` Datetime64,
+ `k6` Float32,
+ `k7` Float64,
+ `k8` Int8,
+ `k9` Int16,
+ `k10` Int32,
+ `k11` Int64,
+ `k12` Int128,
+ `k13` Int256,
+ `k14` UInt8,
+ `k15` UInt16,
+ `k16` UInt32,
+ `k17` UInt64,
+ `k18` UInt128,
+ `k19` UInt256,
+ `k20` Decimal(9,2),
+ `k21` Decimal(18,2),
+ `k22` Decimal(38,2),
+ `k23` Decimal(76,2),
+ `k24` Enum('hello' = 1, 'world' = 2),
+ `k25` IPv4,
+ `k26` IPv6,
+ `k27` UUID,
+ `k28` String,
+ `k29` FixedString(2)
+)
+ENGINE = MergeTree
+ORDER BY k1;
+
+
+
+CREATE TABLE doris_test.student
+(
+ id Int16,
+ name String,
+ age Int16
+)
+ENGINE = MergeTree
+ORDER BY id;
+
+CREATE TABLE doris_test.arr
+(
+ `id` String,
+ `arr1` Array(Bool),
+ `arr2` Array(Date),
+ `arr3` Array(Date32),
+ `arr4` Array(Float32),
+ `arr5` Array(Float64),
+ `arr6` Array(Int8),
+ `arr7` Array(Int16),
+ `arr8` Array(Int32),
+ `arr9` Array(Int64),
+ `arr10` Array(Int128),
+ `arr11` Array(Int256),
+ `arr12` Array(UInt8),
+ `arr13` Array(UInt16),
+ `arr14` Array(UInt32),
+ `arr15` Array(UInt64),
+ `arr16` Array(UInt128),
+ `arr17` Array(UInt256),
+ `arr18` Array(Decimal(9,2)),
+ `arr19` Array(Enum('hello' = 1, 'world' = 2)),
+ `arr20` Array(IPv4),
+ `arr21` Array(IPv6),
+ `arr22` Array(UUID),
+ `arr23` Array(Nullable(Int8)),
+ `arr24` Array(String),
+ `arr25` Array(LowCardinality(String)),
+ `arr26` Array(Datetime),
+ `arr27` Array(Datetime64)
+)
+ENGINE = MergeTree
+ORDER BY id
\ No newline at end of file
diff --git a/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql b/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql
new file mode 100644
index 0000000000..d5df9baba1
--- /dev/null
+++ b/docker/thirdparties/docker-compose/clickhouse/init/04-insert.sql
@@ -0,0 +1,26 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+INSERT INTO doris_test.type VALUES
+(true, '2022-01-01','2022-01-01','2022-01-01 00:00:00','2022-01-01 00:00:00.000000000',1.1,1.1,1,1,1,1,1,1,1,1,1,1,1,1,1.1,1.1,1.1,1.1,1,'116.253.40.133','2a02:aa08:e000:3100::2','61f0c404-5cb3-11e7-907b-a6006ad3dba0','String','F');
+INSERT INTO doris_test.type VALUES
+(false, '2022-01-02','2022-01-02','2022-01-02 00:00:00','2022-01-02 00:00:00.000000000',2.2,2.2,2,2,2,2,2,2,2,2,2,2,2,2,2.2,2.2,2.2,2.2,2,'116.253.40.133','2a02:aa08:e000:3100::2','61f0c404-5cb3-11e7-907b-a6006ad3dba0','String','T');
+
+INSERT INTO doris_test.student values (1, 'doris', 18), (2, 'alice', 19), (3, 'bob', 20);
+
+INSERT INTO doris_test.arr values
+('1',[true],['2022-01-01'],['2022-01-01'],[1.1],[1.1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[1],[2.2],[1],['116.253.40.133'],['2a02:aa08:e000:3100::2'],['61f0c404-5cb3-11e7-907b-a6006ad3dba0'],[1],['string'],['string'],['2022-01-01 00:00:00'],['2022-01-01 00:00:00'])
\ No newline at end of file
diff --git a/docker/thirdparties/run-thirdparties-docker.sh b/docker/thirdparties/run-thirdparties-docker.sh
index bb68a2fe37..aeaad67dff 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -37,7 +37,7 @@ Usage: $0 <options>
--stop stop the specified components
All valid components:
- mysql,pg,oracle,sqlserver,es,hive,iceberg
+ mysql,pg,oracle,sqlserver,clickhouse,es,hive,iceberg
"
exit 1
}
@@ -60,7 +60,7 @@ STOP=0
if [[ "$#" == 1 ]]; then
# default
- COMPONENTS="mysql,pg,oracle,sqlserver,hive,iceberg"
+ COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg"
else
while true; do
case "$1" in
@@ -92,7 +92,7 @@ else
done
if [[ "${COMPONENTS}"x == ""x ]]; then
if [[ "${STOP}" -eq 1 ]]; then
- COMPONENTS="mysql,pg,oracle,sqlserver,hive,iceberg"
+ COMPONENTS="mysql,pg,oracle,sqlserver,clickhouse,hive,iceberg"
fi
fi
fi
@@ -126,6 +126,7 @@ RUN_MYSQL=0
RUN_PG=0
RUN_ORACLE=0
RUN_SQLSERVER=0
+RUN_CLICKHOUSE=0
RUN_HIVE=0
RUN_ES=0
RUN_ICEBERG=0
@@ -138,6 +139,8 @@ for element in "${COMPONENTS_ARR[@]}"; do
RUN_ORACLE=1
elif [[ "${element}"x == "sqlserver"x ]]; then
RUN_SQLSERVER=1
+ elif [[ "${element}"x == "clickhouse"x ]]; then
+ RUN_CLICKHOUSE=1
elif [[ "${element}"x == "es"x ]]; then
RUN_ES=1
elif [[ "${element}"x == "hive"x ]]; then
@@ -216,6 +219,18 @@ if [[ "${RUN_SQLSERVER}" -eq 1 ]]; then
fi
fi
+if [[ "${RUN_CLICKHOUSE}" -eq 1 ]]; then
+ # clickhouse
+ cp "${ROOT}"/docker-compose/clickhouse/clickhouse.yaml.tpl "${ROOT}"/docker-compose/clickhouse/clickhouse.yaml
+ sed -i "s/doris--/${CONTAINER_UID}/g" "${ROOT}"/docker-compose/clickhouse/clickhouse.yaml
+ sudo docker compose -f "${ROOT}"/docker-compose/clickhouse/clickhouse.yaml --env-file "${ROOT}"/docker-compose/clickhouse/clickhouse.env down
+ if [[ "${STOP}" -ne 1 ]]; then
+ sudo mkdir -p "${ROOT}"/docker-compose/clickhouse/data/
+ sudo rm "${ROOT}"/docker-compose/clickhouse/data/* -rf
+ sudo docker compose -f "${ROOT}"/docker-compose/clickhouse/clickhouse.yaml --env-file "${ROOT}"/docker-compose/clickhouse/clickhouse.env up -d
+ fi
+fi
+
if [[ "${RUN_HIVE}" -eq 1 ]]; then
# hive
# before start it, you need to download parquet file package, see "README" in "docker-compose/hive/scripts/"
diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
index 0d284265fb..b1922ac45e 100644
--- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
+++ b/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java
@@ -220,6 +220,7 @@ public class JdbcExecutor {
static {
CK_ARRAY_CONVERTERS.put(String[].class, res -> Arrays.toString((String[]) res));
+ CK_ARRAY_CONVERTERS.put(boolean[].class, res -> Arrays.toString((boolean[]) res));
CK_ARRAY_CONVERTERS.put(byte[].class, res -> Arrays.toString((byte[]) res));
CK_ARRAY_CONVERTERS.put(Byte[].class, res -> Arrays.toString((Byte[]) res));
CK_ARRAY_CONVERTERS.put(LocalDate[].class, res -> Arrays.toString((LocalDate[]) res));
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index ebb52dbe28..fa2a4e3cc2 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -76,6 +76,7 @@ mysql_57_port=3316
pg_14_port=5442
oracle_11_port=1521
sqlserver_2022_port=1433
+clickhouse_22_port=8123
// hive catalog test config
// To enable jdbc test, you need first start hive container.
diff --git a/regression-test/data/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.out b/regression-test/data/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.out
new file mode 100644
index 0000000000..8efde34a30
Binary files /dev/null and b/regression-test/data/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.out differ
diff --git a/regression-test/suites/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.groovy b/regression-test/suites/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.groovy
new file mode 100644
index 0000000000..57d47c9469
--- /dev/null
+++ b/regression-test/suites/jdbc_catalog_p0/test_clickhouse_jdbc_catalog.groovy
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_clickhouse_jdbc_catalog", "p0") {
+ String enabled = context.config.otherConfigs.get("enableJdbcTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String resource_name = "clickhouse_catalog_resource";
+ String catalog_name = "clickhouse_catalog";
+ String internal_db_name = "regression_test_jdbc_catalog_p0";
+ String ex_db_name = "doris_test";
+ String clickhouse_port = context.config.otherConfigs.get("clickhouse_22_port");
+
+ String inDorisTable = "doris_in_tb";
+
+ sql """ drop catalog if exists ${catalog_name} """
+ sql """ drop resource if exists ${resource_name} """
+
+ sql """ create resource if not exists ${resource_name} properties(
+ "type"="jdbc",
+ "user"="default",
+ "password"="123456",
+ "jdbc_url" = "jdbc:clickhouse://127.0.0.1:${clickhouse_port}/doris_test",
+ "driver_url" = "https://doris-community-test-1308700295.cos.ap-hongkong.myqcloud.com/jdbc_driver/clickhouse-jdbc-0.4.2-all.jar",
+ "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver"
+ );"""
+
+ sql """ CREATE CATALOG ${catalog_name} WITH RESOURCE ${resource_name} """
+
+ sql """ drop table if exists ${inDorisTable} """
+ sql """
+ CREATE TABLE ${inDorisTable} (
+ `id` INT NULL COMMENT "主键id",
+ `name` string NULL COMMENT "名字",
+ `age` INT NULL COMMENT "年龄"
+ ) DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+
+ sql """ switch ${catalog_name} """
+ sql """ use ${ex_db_name} """
+
+ order_qt_type """ select * from type order by k1; """
+ order_qt_arr """ select * from arr order by id; """
+ sql """ insert into internal.${internal_db_name}.${inDorisTable} select * from student; """
+ order_qt_in_tb """ select id, name, age from internal.${internal_db_name}.${inDorisTable} order by id; """
+
+ sql """ drop catalog if exists ${catalog_name} """
+ sql """ drop resource if exists ${resource_name} """
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/07: [Enhancement](spark load)Support spark time out config (#17108)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a59478a14146dc1e0e9f4e306b996d4193af0287
Author: liujinhui <96...@qq.com>
AuthorDate: Thu Mar 30 20:12:46 2023 +0800
[Enhancement](spark load)Support spark time out config (#17108)
---
.../doris/load/loadv2/SparkEtlJobHandler.java | 5 ++++-
.../doris/load/loadv2/SparkLauncherMonitor.java | 22 ++++++++++++++++++----
.../load/loadv2/SparkLauncherMonitorTest.java | 7 ++++++-
3 files changed, 28 insertions(+), 6 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
index 57f44f41d5..57f0c58af7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkEtlJobHandler.java
@@ -117,6 +117,8 @@ public class SparkEtlJobHandler {
sparkConfigs.put("spark.yarn.stage.dir", jobStageHdfsPath);
}
+ LOG.info("submit etl spark job, sparkConfigs:{}", sparkConfigs);
+
try {
byte[] configData = etlJobConfig.configToJson().getBytes("UTF-8");
BrokerUtil.writeFile(configData, jobConfigHdfsPath, brokerDesc);
@@ -154,7 +156,8 @@ public class SparkEtlJobHandler {
Process process = launcher.launch();
handle.setProcess(process);
if (!FeConstants.runningUnitTest) {
- SparkLauncherMonitor.LogMonitor logMonitor = SparkLauncherMonitor.createLogMonitor(handle);
+ SparkLauncherMonitor.LogMonitor logMonitor =
+ SparkLauncherMonitor.createLogMonitor(handle, sparkConfigs);
logMonitor.setSubmitTimeoutMs(GET_APPID_TIMEOUT_MS);
logMonitor.setRedirectLogPath(logFilePath);
logMonitor.start();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java
index 012b7ad6d5..4cf388f53b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/SparkLauncherMonitor.java
@@ -20,6 +20,8 @@ package org.apache.doris.load.loadv2;
import com.google.common.base.Preconditions;
import com.google.common.base.Splitter;
import com.google.common.base.Strings;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.logging.log4j.LogManager;
@@ -32,14 +34,15 @@ import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.List;
+import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class SparkLauncherMonitor {
private static final Logger LOG = LogManager.getLogger(SparkLauncherMonitor.class);
- public static LogMonitor createLogMonitor(SparkLoadAppHandle handle) {
- return new LogMonitor(handle);
+ public static LogMonitor createLogMonitor(SparkLoadAppHandle handle, Map<String, String> resourceSparkConfig) {
+ return new LogMonitor(handle, resourceSparkConfig);
}
private static SparkLoadAppHandle.State fromYarnState(YarnApplicationState yarnState) {
@@ -80,18 +83,29 @@ public class SparkLauncherMonitor {
// 5min
private static final long DEFAULT_SUBMIT_TIMEOUT_MS = 300000L;
+ private static final String SUBMIT_TIMEOUT_KEY = "spark.submit.timeout";
- public LogMonitor(SparkLoadAppHandle handle) {
+ public LogMonitor(SparkLoadAppHandle handle, Map<String, String> resourceSparkConfig) {
this.handle = handle;
this.process = handle.getProcess();
this.isStop = false;
- setSubmitTimeoutMs(DEFAULT_SUBMIT_TIMEOUT_MS);
+
+ if (MapUtils.isNotEmpty(resourceSparkConfig)
+ && StringUtils.isNotEmpty(resourceSparkConfig.get(SUBMIT_TIMEOUT_KEY))) {
+ setSubmitTimeoutMs(Long.parseLong(resourceSparkConfig.get(SUBMIT_TIMEOUT_KEY)));
+ } else {
+ setSubmitTimeoutMs(DEFAULT_SUBMIT_TIMEOUT_MS);
+ }
}
public void setSubmitTimeoutMs(long submitTimeoutMs) {
this.submitTimeoutMs = submitTimeoutMs;
}
+ public long getSubmitTimeoutMs() {
+ return submitTimeoutMs;
+ }
+
public void setRedirectLogPath(String redirectLogPath) throws IOException {
this.outputStream = new FileOutputStream(new File(redirectLogPath), false);
this.handle.setLogPath(redirectLogPath);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLauncherMonitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLauncherMonitorTest.java
index f9874c33c7..c9f28f188e 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLauncherMonitorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/load/loadv2/SparkLauncherMonitorTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.load.loadv2;
+import org.apache.commons.collections.map.HashedMap;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.junit.After;
import org.junit.Assert;
@@ -26,6 +27,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.util.Map;
public class SparkLauncherMonitorTest {
private String appId;
@@ -51,13 +53,16 @@ public class SparkLauncherMonitorTest {
@Test
public void testLogMonitorNormal() {
+ Map<String, String> config = new HashedMap();
+ config.put("spark.submit.timeout", "600000");
URL log = getClass().getClassLoader().getResource("spark_launcher_monitor.log");
String cmd = "cat " + log.getPath();
SparkLoadAppHandle handle = null;
try {
Process process = Runtime.getRuntime().exec(cmd);
handle = new SparkLoadAppHandle(process);
- SparkLauncherMonitor.LogMonitor logMonitor = SparkLauncherMonitor.createLogMonitor(handle);
+ SparkLauncherMonitor.LogMonitor logMonitor = SparkLauncherMonitor.createLogMonitor(handle, config);
+ Assert.assertEquals(logMonitor.getSubmitTimeoutMs(), 600000L);
logMonitor.setRedirectLogPath(logPath);
logMonitor.start();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/07: [Fix](multi-catalog) fix hms automatic update error. (#18252)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1b2096502b682da77551938fca7099e79787319a
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Thu Mar 30 23:09:07 2023 +0800
[Fix](multi-catalog) fix hms automatic update error. (#18252)
Co-authored-by: wangxiangyu@360shuke.com <wa...@360shuke.com>
---
.../src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
index 0cf7afdc53..1f0b65c720 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
@@ -52,7 +52,9 @@ public class HMSExternalCatalog extends ExternalCatalog {
private static final int MAX_CLIENT_POOL_SIZE = 8;
protected PooledHiveMetaStoreClient client;
// Record the latest synced event id when processing hive events
- private long lastSyncedEventId;
+ // Must set to -1 otherwise client.getNextNotification will throw exception
+ // Reference to https://github.com/apache/doris/issues/18251
+ private long lastSyncedEventId = -1L;
/**
* Default constructor for HMSExternalCatalog.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/07: [log](load) print detailed error message when publish failed (#18176)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit b768b8358353aec1103ed7e6cf2a791feb1c1280
Author: zhannngchen <48...@users.noreply.github.com>
AuthorDate: Wed Mar 29 09:14:47 2023 +0800
[log](load) print detailed error message when publish failed (#18176)
W0326 22:46:21.081120 30803 engine_publish_version_task.cpp:215] failed to publish version. rowset_id=0200000000010b38154fe7bb7ca4314a18698cb6a0efc9a3, tablet_id=63390785, txn_id=43714869
The log missed detailed error message
---
be/src/olap/task/engine_publish_version_task.cpp | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/task/engine_publish_version_task.cpp b/be/src/olap/task/engine_publish_version_task.cpp
index f8847c173c..5d3a65cf3c 100644
--- a/be/src/olap/task/engine_publish_version_task.cpp
+++ b/be/src/olap/task/engine_publish_version_task.cpp
@@ -217,7 +217,8 @@ void TabletPublishTxnTask::handle() {
_partition_id, _tablet, _transaction_id, _version);
if (publish_status != Status::OK()) {
LOG(WARNING) << "failed to publish version. rowset_id=" << _rowset->rowset_id()
- << ", tablet_id=" << _tablet_info.tablet_id << ", txn_id=" << _transaction_id;
+ << ", tablet_id=" << _tablet_info.tablet_id << ", txn_id=" << _transaction_id
+ << ", res=" << publish_status;
_engine_publish_version_task->add_error_tablet_id(_tablet_info.tablet_id);
return;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/07: [fix](memory) Remove page cache regular clear, disabled jemalloc prof by default (#18218)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c3485b9fc3e7786bb361fc98b2fbe21cd6a23de8
Author: Xinyi Zou <zo...@gmail.com>
AuthorDate: Thu Mar 30 09:39:37 2023 +0800
[fix](memory) Remove page cache regular clear, disabled jemalloc prof by default (#18218)
Remove page cache regular clear
Now the page cache is turned off by default. If the user manually opens the page cache, it can be considered that the user can accept the memory usage of the page cache, and then can consider adding a manual clear command to the cache.
fix memory gc cancel top memory query
jemalloc prof is not enabled by default
---
be/src/common/config.h | 4 ---
be/src/common/daemon.cpp | 12 ---------
be/src/runtime/memory/mem_tracker_limiter.cpp | 5 +---
bin/start_be.sh | 4 +--
docs/en/community/developer-guide/be-vscode-dev.md | 2 ++
docs/en/community/developer-guide/debug-tool.md | 31 ++++++++++++----------
.../community/developer-guide/be-vscode-dev.md | 2 ++
docs/zh-CN/community/developer-guide/debug-tool.md | 2 +-
8 files changed, 25 insertions(+), 37 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 95db7aecd8..4bdc1dfebf 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -74,10 +74,6 @@ CONF_Int64(max_sys_mem_available_low_water_mark_bytes, "1717986918");
// The size of the memory that gc wants to release each time, as a percentage of the mem limit.
CONF_mString(process_minor_gc_size, "10%");
CONF_mString(process_full_gc_size, "20%");
-// Some caches have their own gc threads, such as segment cache.
-// For caches that do not have a separate gc thread, perform regular gc in the memory maintenance thread.
-// Currently only storage page cache, chunk allocator, more in the future.
-CONF_mInt32(cache_gc_interval_s, "60");
// If true, when the process does not exceed the soft mem limit, the query memory will not be limited;
// when the process memory exceeds the soft mem limit, the query with the largest ratio between the currently
diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
index f45eb1700e..322b19b159 100644
--- a/be/src/common/daemon.cpp
+++ b/be/src/common/daemon.cpp
@@ -241,10 +241,8 @@ void Daemon::memory_maintenance_thread() {
void Daemon::memory_gc_thread() {
int32_t interval_milliseconds = config::memory_maintenance_sleep_time_ms;
- int32_t cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
int32_t memory_minor_gc_sleep_time_ms = 0;
int32_t memory_full_gc_sleep_time_ms = 0;
- int64_t cache_gc_freed_mem = 0;
while (!_stop_background_threads_latch.wait_for(
std::chrono::milliseconds(interval_milliseconds))) {
if (!MemInfo::initialized() || !ExecEnv::GetInstance()->initialized()) {
@@ -257,7 +255,6 @@ void Daemon::memory_gc_thread() {
// No longer full gc and minor gc during sleep.
memory_full_gc_sleep_time_ms = config::memory_gc_sleep_time_s * 1000;
memory_minor_gc_sleep_time_ms = config::memory_gc_sleep_time_s * 1000;
- cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
doris::MemTrackerLimiter::print_log_process_usage("process full gc", false);
if (doris::MemInfo::process_full_gc()) {
// If there is not enough memory to be gc, the process memory usage will not be printed in the next continuous gc.
@@ -270,7 +267,6 @@ void Daemon::memory_gc_thread() {
doris::MemInfo::soft_mem_limit())) {
// No minor gc during sleep, but full gc is possible.
memory_minor_gc_sleep_time_ms = config::memory_gc_sleep_time_s * 1000;
- cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
doris::MemTrackerLimiter::print_log_process_usage("process minor gc", false);
if (doris::MemInfo::process_minor_gc()) {
doris::MemTrackerLimiter::enable_print_log_process_usage();
@@ -282,14 +278,6 @@ void Daemon::memory_gc_thread() {
if (memory_minor_gc_sleep_time_ms > 0) {
memory_minor_gc_sleep_time_ms -= interval_milliseconds;
}
- cache_gc_interval_ms -= interval_milliseconds;
- if (cache_gc_interval_ms < 0) {
- cache_gc_freed_mem = 0;
- doris::MemInfo::process_cache_gc(cache_gc_freed_mem);
- LOG(INFO) << fmt::format("Process regular GC Cache, Free Memory {} Bytes",
- cache_gc_freed_mem);
- cache_gc_interval_ms = config::cache_gc_interval_s * 1000;
- }
}
}
}
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 0a054a7dfe..e7f15bf212 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -305,9 +305,6 @@ int64_t MemTrackerLimiter::free_top_memory_query(int64_t min_free_mem,
std::lock_guard<std::mutex> l(mem_tracker_limiter_pool[i].group_lock);
for (auto tracker : mem_tracker_limiter_pool[i].trackers) {
if (tracker->type() == type) {
- if (tracker->consumption() <= 104857600) { // 100M small query does not cancel
- continue;
- }
if (ExecEnv::GetInstance()->fragment_mgr()->query_is_canceled(
label_to_queryid(tracker->label()))) {
continue;
@@ -351,7 +348,7 @@ int64_t MemTrackerLimiter::free_top_overcommit_query(int64_t min_free_mem,
std::lock_guard<std::mutex> l(mem_tracker_limiter_pool[i].group_lock);
for (auto tracker : mem_tracker_limiter_pool[i].trackers) {
if (tracker->type() == type) {
- if (tracker->consumption() <= 104857600) { // 100M small query does not cancel
+ if (tracker->consumption() <= 33554432) { // 32M small query does not cancel
continue;
}
if (ExecEnv::GetInstance()->fragment_mgr()->query_is_canceled(
diff --git a/bin/start_be.sh b/bin/start_be.sh
index d383c25163..89c40947c0 100755
--- a/bin/start_be.sh
+++ b/bin/start_be.sh
@@ -235,8 +235,8 @@ if [[ -f "${DORIS_HOME}/conf/hdfs-site.xml" ]]; then
export LIBHDFS3_CONF="${DORIS_HOME}/conf/hdfs-site.xml"
fi
-# see https://github.com/jemalloc/jemalloc/issues/2366
-export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:30000,dirty_decay_ms:30000,oversize_threshold:0,lg_tcache_max:16,prof:true,prof_prefix:jeprof.out"
+# see https://github.com/apache/doris/blob/master/docs/zh-CN/community/developer-guide/debug-tool.md#jemalloc-heap-profile
+export JEMALLOC_CONF="percpu_arena:percpu,background_thread:true,metadata_thp:auto,muzzy_decay_ms:30000,dirty_decay_ms:30000,oversize_threshold:0,lg_tcache_max:16,prof_prefix:jeprof.out"
if [[ "${RUN_DAEMON}" -eq 1 ]]; then
nohup ${LIMIT:+${LIMIT}} "${DORIS_HOME}/lib/doris_be" "$@" >>"${LOG_DIR}/be.out" 2>&1 </dev/null &
diff --git a/docs/en/community/developer-guide/be-vscode-dev.md b/docs/en/community/developer-guide/be-vscode-dev.md
index 0c71d6f575..ce2b1c083f 100644
--- a/docs/en/community/developer-guide/be-vscode-dev.md
+++ b/docs/en/community/developer-guide/be-vscode-dev.md
@@ -207,6 +207,8 @@ In the configuration **"request": "attach", "processId": PID**, these two config
ps -ef | grep palo*
```
+Or write **"processId": "${command:pickProcess}"** to specify the pid when starting attach.
+
As shown in the figure:
diff --git a/docs/en/community/developer-guide/debug-tool.md b/docs/en/community/developer-guide/debug-tool.md
index 96c705b454..9d14782ab5 100644
--- a/docs/en/community/developer-guide/debug-tool.md
+++ b/docs/en/community/developer-guide/debug-tool.md
@@ -238,7 +238,7 @@ From the above output, we can see that 1024 bytes have been leaked, and the stac
#### JEMALLOC HEAP PROFILE
##### 1. runtime heap dump by http
-No need to restart BE, use jemalloc heap dump http interface, jemalloc generates heap dump file on the corresponding BE machine according to the current memory usage.
+Add `,prof:true,lg_prof_sample:10` to `JEMALLOC_CONF` in `start_be.sh` and restart BE, then use the jemalloc heap dump http interface to generate a heap dump file on the corresponding BE machine.
The directory where the heap dump file is located can be configured through the ``jeprofile_dir`` variable in ``be.conf``, and the default is ``${DORIS_HOME}/log``
@@ -246,21 +246,12 @@ The directory where the heap dump file is located can be configured through the
curl http://be_host:be_webport/jeheap/dump
```
-##### 2. heap dump by JEMALLOC_CONF
-Perform heap dump by restarting BE after changing the `JEMALLOC_CONF` variable in `start_be.sh`
-
-1. Dump every 1MB:
-
- Two new variable settings `prof:true,lg_prof_interval:20` have been added to the `JEMALLOC_CONF` variable, where `prof:true` is to enable profiling, and `lg_prof_interval:20` means that a dump is generated every 1MB (2^20)
-2. Dump each time a new high is reached:
-
- Added two variable settings `prof:true,prof_gdump:true` in the `JEMALLOC_CONF` variable, where `prof:true` is to enable profiling, and `prof_gdump:true` means to generate a dump when the memory usage reaches a new high
-3. Memory leak dump when the program exits:
-
- Added three new variable settings `prof_leak: true, lg_prof_sample: 0, prof_final: true` in the `JEMALLOC_CONF` variable
+`prof`: After opening, jemalloc will generate a heap dump file according to the current memory usage. There is a small amount of performance loss in heap profile sampling, which can be turned off during performance testing.
+`lg_prof_sample`: heap profile sampling interval, the default value is 19, that is, the default sampling interval is 512K (2^19 B), which will result in only 10% of the memory recorded by the heap profile, `lg_prof_sample:10` can reduce the sampling interval to 1K (2^10 B), more frequent sampling will make the heap profile close to real memory, but this will bring greater performance loss.
+For detailed parameter description, refer to https://linux.die.net/man/3/jemalloc.
-#### 3. jemalloc heap dump profiling
+#### 2. jemalloc heap dump profiling
3.1 Generating plain text analysis results
```shell
@@ -287,6 +278,18 @@ jeprof lib/doris_be --base=heap_dump_file_1 heap_dump_file_2
In the above jeprof related commands, remove the `--base` option to analyze only a single heap dump file
+##### 3. heap dump by JEMALLOC_CONF
+Perform heap dump by restarting BE after changing the `JEMALLOC_CONF` variable in `start_be.sh`
+
+1. Dump every 1MB:
+
+ Two new variable settings `prof:true,lg_prof_interval:20` have been added to the `JEMALLOC_CONF` variable, where `prof:true` is to enable profiling, and `lg_prof_interval:20` means that a dump is generated every 1MB (2^20)
+2. Dump each time a new high is reached:
+
+ Added two variable settings `prof:true,prof_gdump:true` in the `JEMALLOC_CONF` variable, where `prof:true` is to enable profiling, and `prof_gdump:true` means to generate a dump when the memory usage reaches a new high
+3. Memory leak dump when the program exits:
+
+ Added three new variable settings `prof_leak: true, lg_prof_sample: 0, prof_final: true` in the `JEMALLOC_CONF` variable
#### ASAN
diff --git a/docs/zh-CN/community/developer-guide/be-vscode-dev.md b/docs/zh-CN/community/developer-guide/be-vscode-dev.md
index 4bcaf5d407..338413dc38 100644
--- a/docs/zh-CN/community/developer-guide/be-vscode-dev.md
+++ b/docs/zh-CN/community/developer-guide/be-vscode-dev.md
@@ -206,6 +206,8 @@ mkdir -p /soft/be/storage
ps -ef | grep palo*
```
+或者写作 **"processId": "${command:pickProcess}"**，可在启动attach时指定pid.
+
如图：
diff --git a/docs/zh-CN/community/developer-guide/debug-tool.md b/docs/zh-CN/community/developer-guide/debug-tool.md
index b4952f9a13..e47c5a0047 100644
--- a/docs/zh-CN/community/developer-guide/debug-tool.md
+++ b/docs/zh-CN/community/developer-guide/debug-tool.md
@@ -202,7 +202,7 @@ Total: 1296.4 MB
#### JEMALLOC HEAP PROFILE
##### 1. runtime heap dump by http
-无需重启BE, 使用jemalloc heap dump http接口，jemalloc根据当前内存使用情况，在对应的BE机器上生成heap dump文件。
+在`start_be.sh` 中`JEMALLOC_CONF` 增加 `,prof:true,lg_prof_sample:10` 并重启BE，然后使用jemalloc heap dump http接口，在对应的BE机器上生成heap dump文件。
heap dump文件所在目录可以在 ``be.conf`` 中通过``jeprofile_dir``变量进行配置，默认为``${DORIS_HOME}/log``
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/07: [fix](planner )need add LateralViewRef's id into TableRef's allTableRefIds (#18220)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 51741e886e42de2eb0a028cd4c393cbb556891c0
Author: starocean999 <40...@users.noreply.github.com>
AuthorDate: Thu Mar 30 11:32:18 2023 +0800
[fix](planner )need add LateralViewRef's id into TableRef's allTableRefIds (#18220)
1. add LateralViewRef's id into TableRef's allTableRefIds, so the caller won't miss LateralViewRef when trying to get all the tableref ids.
2. TableFunctionNode should use child node's output tuple id as the input tuple id
---
.../java/org/apache/doris/analysis/SelectStmt.java | 12 +++-
.../java/org/apache/doris/analysis/TableRef.java | 6 ++
.../org/apache/doris/planner/AnalyticPlanner.java | 2 +-
.../apache/doris/planner/TableFunctionNode.java | 2 +-
.../test_inlineview_with_window_function.groovy | 73 ++++++++++++++++++++++
5 files changed, 92 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index b3fab0ccea..0baffbc15f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -596,6 +596,16 @@ public class SelectStmt extends QueryStmt {
return result;
}
+ public List<TupleId> getAllTableRefIds() {
+ List<TupleId> result = Lists.newArrayList();
+
+ for (TableRef ref : fromClause) {
+ result.addAll(ref.getAllTableRefIds());
+ }
+
+ return result;
+ }
+
public List<TupleId> getTableRefIdsWithoutInlineView() {
List<TupleId> result = Lists.newArrayList();
@@ -696,7 +706,7 @@ public class SelectStmt extends QueryStmt {
// can also be safely evaluated below the join (picked up by getBoundPredicates()).
// Such predicates will be marked twice and that is ok.
List<Expr> unassigned =
- analyzer.getUnassignedConjuncts(getTableRefIds(), true);
+ analyzer.getUnassignedConjuncts(getAllTableRefIds(), true);
List<Expr> unassignedJoinConjuncts = Lists.newArrayList();
for (Expr e : unassigned) {
if (analyzer.evalAfterJoin(e)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
index 1caa0bba97..7e9deadd84 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
@@ -681,6 +681,12 @@ public class TableRef implements ParseNode, Writable {
// Indicate that this table ref has an empty ON-clause.
analyzer.registerOnClauseConjuncts(Collections.<Expr>emptyList(), this);
}
+
+ if (lateralViewRefs != null) {
+ for (LateralViewRef lateralViewRef : lateralViewRefs) {
+ allTableRefIds.add(lateralViewRef.getId());
+ }
+ }
}
public void rewriteExprs(ExprRewriter rewriter, Analyzer analyzer)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java
index e7e7533ff6..33417662df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/AnalyticPlanner.java
@@ -301,7 +301,7 @@ public class AnalyticPlanner {
ExprSubstitutionMap sortSmap = new ExprSubstitutionMap();
List<Expr> sortSlotExprs = Lists.newArrayList();
sortTupleDesc.setIsMaterialized(true);
- for (TupleId tid : input.getTupleIds()) {
+ for (TupleId tid : input.getOutputTupleIds()) {
TupleDescriptor tupleDesc = analyzer.getTupleDesc(tid);
for (SlotDescriptor inputSlotDesc : tupleDesc.getSlots()) {
if (!inputSlotDesc.isMaterialized()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
index 592621e54d..4c21a89e5d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
@@ -52,7 +52,7 @@ public class TableFunctionNode extends PlanNode {
protected TableFunctionNode(PlanNodeId id, PlanNode inputNode, List<LateralViewRef> lateralViewRefs) {
super(id, "TABLE FUNCTION NODE", StatisticalType.TABLE_FUNCTION_NODE);
- tupleIds.addAll(inputNode.getTupleIds());
+ tupleIds.addAll(inputNode.getOutputTupleIds());
tblRefIds.addAll(inputNode.getTupleIds());
tblRefIds.addAll(inputNode.getTblRefIds());
lateralViewTupleIds = lateralViewRefs.stream().map(e -> e.getDesc().getId())
diff --git a/regression-test/suites/correctness_p0/test_inlineview_with_window_function.groovy b/regression-test/suites/correctness_p0/test_inlineview_with_window_function.groovy
new file mode 100644
index 0000000000..0da26ff8f3
--- /dev/null
+++ b/regression-test/suites/correctness_p0/test_inlineview_with_window_function.groovy
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_inlineview_with_window_function") {
+ sql """
+ drop view if exists test_table_aa;
+ """
+
+ sql """
+ drop table if exists test_table_bb;
+ """
+
+ sql """
+ CREATE TABLE `test_table_bb` (
+ `event_date` datetime NULL,
+ `event_content` text NULL,
+ `animal_id` bigint(20) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`event_date`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`event_date`) BUCKETS 48
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "false",
+ "storage_format" = "V2",
+ "disable_auto_compaction" = "false"
+ );
+ """
+
+ sql """
+ CREATE VIEW test_table_aa AS
+ SELECT
+ ev.event_date,
+ ev.animal_id,
+ get_json_int(ev.event_content, "\$.nurseOns") as nurseons
+ FROM test_table_bb ev;
+ """
+
+ sql """
+ SELECT row_number() over(PARTITION BY animal_id ORDER BY event_date DESC) rw
+ FROM test_table_aa;
+ """
+
+ sql """
+ select
+ e1
+ from test_table_aa
+ lateral view explode([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23]) tmp1 as e1
+ where animal_id <= e1;
+ """
+
+ sql """
+ drop view if exists test_table_aa;
+ """
+
+ sql """
+ drop table if exists test_table_bb;
+ """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org