You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/07 00:58:10 UTC
[doris] branch branch-1.2-lts updated (f3f83932a6 -> 50f4a73564)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
from f3f83932a6 [fix](hudi)degrade hudi version for 1.2 to fix unexpected exception (#18426)
new e2bbd2efef [improvement](coverage) build be with coverage enabled, which can get coverage data with llvm-cov-15 (#17995)
new 2ffffbc106 [Bug](DECIMALV3) Fix wrong precision for plus/minus (#18052)
new c4c9e1c1c3 [Chore](planner) add error information for OnClause contain ExistsPredicates (#18090)
new 069b0a800e [regression-test](log) add log for malforamt response of stream load (#18173)
new 8f8adea52b [Fix](planner) fix nested udf bind arguments exception (#18188)
new ad052f8e47 [Fix](planner)fix create view ignore order by info bug. (#18197)
new 50f4a73564 [Enhancement](spark load)Support for RM HA (#15000)
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/CMakeLists.txt | 5 +
be/src/vec/data_types/data_type_decimal.h | 2 +-
build.sh | 9 ++
.../main/java/org/apache/doris/catalog/Type.java | 23 +++++
.../org/apache/doris/analysis/ArithmeticExpr.java | 6 +-
.../java/org/apache/doris/analysis/CastExpr.java | 2 +-
.../java/org/apache/doris/analysis/FromClause.java | 5 +
.../apache/doris/analysis/FunctionCallExpr.java | 3 +
.../java/org/apache/doris/analysis/SelectStmt.java | 9 +-
.../org/apache/doris/catalog/SparkResource.java | 35 ++++++-
.../org/apache/doris/catalog/CreateViewTest.java | 2 +-
.../apache/doris/catalog/SparkResourceTest.java | 16 ++++
.../decimalv3/test_arithmetic_expressions.out | 15 +++
.../regression/action/StreamLoadAction.groovy | 54 ++++++-----
.../decimalv3/test_arithmetic_expressions.groovy | 16 ++++
.../suites/query_p0/join/test_join.groovy | 5 +-
.../test_alias_function.groovy} | 20 ++--
.../suites/query_p0/view/test_view.groovy | 104 +++++++++++++++++++++
run-be-ut.sh | 22 ++++-
19 files changed, 301 insertions(+), 52 deletions(-)
copy regression-test/suites/query_p0/{explain/test_pushdown_explain.groovy => sql_functions/test_alias_function.groovy} (57%)
create mode 100644 regression-test/suites/query_p0/view/test_view.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/07: [improvement](coverage) build be with coverage enabled, which can get coverage data with llvm-cov-15 (#17995)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e2bbd2efef0c069bdadc91c2cdf537a2805b0217
Author: chunping <ce...@163.com>
AuthorDate: Thu Mar 23 12:07:19 2023 +0800
[improvement](coverage) build be with coverage enabled, which can get coverage data with llvm-cov-15 (#17995)
---
be/CMakeLists.txt | 5 +++++
build.sh | 9 +++++++++
run-be-ut.sh | 22 ++++++++++++++++++++--
3 files changed, 34 insertions(+), 2 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index aedb3a97b7..114e7be057 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -856,6 +856,11 @@ endif()
# Only build static libs
set(BUILD_SHARED_LIBS OFF)
+option(ENABLE_CLANG_COVERAGE "coverage option" OFF)
+if (ENABLE_CLANG_COVERAGE AND ENABLE_CLANG_COVERAGE STREQUAL ON AND COMPILER_CLANG)
+ SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
+endif ()
+
if (${MAKE_TEST} STREQUAL "ON")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage -DGTEST_USE_OWN_TR1_TUPLE=0")
SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
diff --git a/build.sh b/build.sh
index 17f6b96ea6..9eaa16d8b6 100755
--- a/build.sh
+++ b/build.sh
@@ -63,6 +63,7 @@ Usage: $0 <options>
$0 --spark-dpp build Spark DPP application alone
$0 --broker build Broker
$0 --be --fe build Backend, Frontend, Spark Dpp application and Java UDF library
+ $0 --be --coverage build Backend with coverage enabled
USE_AVX2=0 $0 --be build Backend and not using AVX2 instruction.
USE_AVX2=0 STRIP_DEBUG_INFO=ON $0 build all and not using AVX2 instruction, and strip the debug info for Backend
@@ -115,6 +116,7 @@ if ! OPTS="$(getopt \
-l 'spark-dpp' \
-l 'hive-udf' \
-l 'clean' \
+ -l 'coverage' \
-l 'help' \
-o 'hj:' \
-- "$@")"; then
@@ -136,6 +138,7 @@ CLEAN=0
HELP=0
PARAMETER_COUNT="$#"
PARAMETER_FLAG=0
+DENABLE_CLANG_COVERAGE='OFF'
if [[ "$#" == 1 ]]; then
# default
BUILD_FE=1
@@ -186,6 +189,10 @@ else
CLEAN=1
shift
;;
+ --coverage)
+ DENABLE_CLANG_COVERAGE='ON'
+ shift
+ ;;
-h)
HELP=1
shift
@@ -350,6 +357,7 @@ echo "Get params:
USE_JEMALLOC -- ${USE_JEMALLOC}
STRICT_MEMORY_USE -- ${STRICT_MEMORY_USE}
ENABLE_STACKTRACE -- ${ENABLE_STACKTRACE}
+ DENABLE_CLANG_COVERAGE -- ${DENABLE_CLANG_COVERAGE}
"
# Clean and build generated code
@@ -424,6 +432,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DUSE_AVX2="${USE_AVX2}" \
-DGLIBC_COMPATIBILITY="${GLIBC_COMPATIBILITY}" \
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
+ -DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
"${DORIS_HOME}/be"
"${BUILD_SYSTEM}" -j "${PARALLEL}"
"${BUILD_SYSTEM}" install
diff --git a/run-be-ut.sh b/run-be-ut.sh
index eb695545dc..91cf61148a 100755
--- a/run-be-ut.sh
+++ b/run-be-ut.sh
@@ -50,6 +50,7 @@ Usage: $0 <options>
--clean clean and build ut
--run build and run all ut
--run --filter=xx build and run specified ut
+ --coverage coverage after run ut
-j build parallel
-h print this help message
@@ -64,11 +65,12 @@ Usage: $0 <options>
$0 --run --filter=FooTest.*:BarTest.*-FooTest.Bar:BarTest.Foo runs everything in test suite FooTest except FooTest.Bar and everything in test suite BarTest except BarTest.Foo
$0 --clean clean and build tests
$0 --clean --run clean, build and run all tests
+ $0 --clean --run --coverage clean, build, run all tests and coverage
"
exit 1
}
-if ! OPTS="$(getopt -n "$0" -o vhj:f: -l benchmark,run,clean,filter: -- "$@")"; then
+if ! OPTS="$(getopt -n "$0" -o vhj:f: -l coverage,benchmark,run,clean,filter: -- "$@")"; then
usage
fi
@@ -77,6 +79,7 @@ eval set -- "${OPTS}"
CLEAN=0
RUN=0
BUILD_BENCHMARK_TOOL='OFF'
+DENABLE_CLANG_COVERAGE='OFF'
FILTER=""
if [[ "$#" != 1 ]]; then
while true; do
@@ -93,6 +96,10 @@ if [[ "$#" != 1 ]]; then
BUILD_BENCHMARK_TOOL='ON'
shift
;;
+ --coverage)
+ DENABLE_CLANG_COVERAGE='ON'
+ shift
+ ;;
-f | --filter)
FILTER="--gtest_filter=$2"
shift 2
@@ -125,6 +132,10 @@ echo "Get params:
"
echo "Build Backend UT"
+if [[ "_${DENABLE_CLANG_COVERAGE}" == "_ON" ]]; then
+ echo "export DORIS_TOOLCHAIN=clang" >>custom_env.sh
+fi
+
CMAKE_BUILD_DIR="${DORIS_HOME}/be/ut_build_${CMAKE_BUILD_TYPE}"
if [[ "${CLEAN}" -eq 1 ]]; then
pushd "${DORIS_HOME}/gensrc"
@@ -187,6 +198,7 @@ cd "${CMAKE_BUILD_DIR}"
-DUSE_JEMALLOC=OFF \
-DSTRICT_MEMORY_USE=OFF \
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
+ -DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
"${DORIS_HOME}/be"
"${BUILD_SYSTEM}" -j "${PARALLEL}"
@@ -267,9 +279,15 @@ export UBSAN_OPTIONS=print_stacktrace=1
# find all executable test files
test="${DORIS_TEST_BINARY_DIR}/doris_be_test"
+profraw=${DORIS_TEST_BINARY_DIR}/doris_be_test.profraw
+
file_name="${test##*/}"
if [[ -f "${test}" ]]; then
- "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}"
+ if [[ "_${DENABLE_CLANG_COVERAGE}" == "_ON" ]]; then
+ LLVM_PROFILE_FILE="${profraw}" "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}"
+ else
+ "${test}" --gtest_output="xml:${GTEST_OUTPUT_DIR}/${file_name}.xml" --gtest_print_time=true "${FILTER}"
+ fi
echo "=== Finished. Gtest output: ${GTEST_OUTPUT_DIR}"
else
echo "unit test file: ${test} does not exist."
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/07: [Bug](DECIMALV3) Fix wrong precision for plus/minus (#18052)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2ffffbc1065d788ae7ac2d3591c3fc0f27f7aa60
Author: Gabriel <ga...@gmail.com>
AuthorDate: Sat Mar 25 09:42:39 2023 +0800
[Bug](DECIMALV3) Fix wrong precision for plus/minus (#18052)
Result type for DECIMAL(x, y) plus/minus DECIMAL(m, n) should be DECIMAL(max(x - y, m - n) + max(y + n) + 1, max(y + n))
---
be/src/vec/data_types/data_type_decimal.h | 2 +-
.../main/java/org/apache/doris/catalog/Type.java | 23 ++++++++++++++++++++++
.../org/apache/doris/analysis/ArithmeticExpr.java | 6 +++---
.../java/org/apache/doris/analysis/CastExpr.java | 2 +-
.../decimalv3/test_arithmetic_expressions.out | 15 ++++++++++++++
.../decimalv3/test_arithmetic_expressions.groovy | 16 +++++++++++++++
6 files changed, 59 insertions(+), 5 deletions(-)
diff --git a/be/src/vec/data_types/data_type_decimal.h b/be/src/vec/data_types/data_type_decimal.h
index 358fe79438..c644fbcfc0 100644
--- a/be/src/vec/data_types/data_type_decimal.h
+++ b/be/src/vec/data_types/data_type_decimal.h
@@ -236,7 +236,7 @@ DataTypePtr decimal_result_type(const DataTypeDecimal<T>& tx, const DataTypeDeci
size_t divide_precision = tx.get_precision() + ty.get_scale();
size_t plus_minus_precision =
std::max(tx.get_precision() - tx.get_scale(), ty.get_precision() - ty.get_scale()) +
- scale;
+ scale + 1;
if (is_multiply) {
scale = tx.get_scale() + ty.get_scale();
precision = std::min(multiply_precision, max_decimal_precision<Decimal128I>());
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index f5d02f5ae1..671da2c252 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -1713,6 +1713,10 @@ public abstract class Type {
// Whether `type1` matches the exact type of `type2`.
public static boolean matchExactType(Type type1, Type type2) {
+ return matchExactType(type1, type2, false);
+ }
+
+ public static boolean matchExactType(Type type1, Type type2, boolean ignorePrecision) {
if (type1.matchesType(type2)) {
if (PrimitiveType.typeWithPrecision.contains(type2.getPrimitiveType())) {
// For types which has precision and scale, we also need to check quality between precisions and scales
@@ -1720,6 +1724,10 @@ public abstract class Type {
== ((ScalarType) type1).decimalPrecision()) && (((ScalarType) type2).decimalScale()
== ((ScalarType) type1).decimalScale())) {
return true;
+ } else if (((ScalarType) type2).decimalScale() == ((ScalarType) type1).decimalScale()
+ && ignorePrecision) {
+ return isSameDecimalTypeWithDifferentPrecision(((ScalarType) type2).decimalPrecision(),
+ ((ScalarType) type1).decimalPrecision());
}
} else if (type2.isArrayType()) {
// For types array, we also need to check contains null for case like
@@ -1733,5 +1741,20 @@ public abstract class Type {
}
return false;
}
+
+ public static boolean isSameDecimalTypeWithDifferentPrecision(int precision1, int precision2) {
+ if (precision1 <= ScalarType.MAX_DECIMAL32_PRECISION && precision2 <= ScalarType.MAX_DECIMAL32_PRECISION) {
+ return true;
+ } else if (precision1 > ScalarType.MAX_DECIMAL32_PRECISION && precision2 > ScalarType.MAX_DECIMAL32_PRECISION
+ && precision1 <= ScalarType.MAX_DECIMAL64_PRECISION
+ && precision2 <= ScalarType.MAX_DECIMAL64_PRECISION) {
+ return true;
+ } else if (precision1 > ScalarType.MAX_DECIMAL64_PRECISION && precision2 > ScalarType.MAX_DECIMAL64_PRECISION
+ && precision1 <= ScalarType.MAX_DECIMAL128_PRECISION
+ && precision2 <= ScalarType.MAX_DECIMAL128_PRECISION) {
+ return true;
+ }
+ return false;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
index 02ea5579cb..a6ab76df9c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ArithmeticExpr.java
@@ -532,7 +532,7 @@ public class ArithmeticExpr extends Expr {
// target type: DECIMALV3(max(widthOfIntPart1, widthOfIntPart2) + max(scale1, scale2) + 1,
// max(scale1, scale2))
scale = Math.max(t1Scale, t2Scale);
- precision = Math.max(widthOfIntPart1, widthOfIntPart2) + scale;
+ precision = Math.max(widthOfIntPart1, widthOfIntPart2) + scale + 1;
} else {
scale = Math.max(t1Scale, t2Scale);
precision = widthOfIntPart2 + scale;
@@ -547,10 +547,10 @@ public class ArithmeticExpr extends Expr {
}
type = ScalarType.createDecimalV3Type(precision, scale);
if (op == Operator.ADD || op == Operator.SUBTRACT) {
- if (!Type.matchExactType(type, children.get(0).type)) {
+ if (((ScalarType) type).getScalarScale() != ((ScalarType) children.get(0).type).getScalarScale()) {
castChild(type, 0);
}
- if (!Type.matchExactType(type, children.get(1).type)) {
+ if (((ScalarType) type).getScalarScale() != ((ScalarType) children.get(1).type).getScalarScale()) {
castChild(type, 1);
}
} else if (op == Operator.DIVIDE && (t2Scale != 0) && t1.isDecimalV3()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index d2cb8ce442..e364de5279 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -281,7 +281,7 @@ public class CastExpr extends Expr {
Type childType = getChild(0).getType();
// this cast may result in loss of precision, but the user requested it
- noOp = Type.matchExactType(childType, type);
+ noOp = Type.matchExactType(childType, type, true);
if (noOp) {
// For decimalv2, we do not perform an actual cast between different precision/scale. Instead, we just
diff --git a/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out b/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out
index 4f68777f2a..085b844d7c 100644
--- a/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out
+++ b/regression-test/data/datatype_p0/decimalv3/test_arithmetic_expressions.out
@@ -32,3 +32,18 @@
2.0736
3.2399999999999998
+-- !select_all --
+999999.999 999999.999 999999.999 999999.999 999999.999 999999.999 999999.999 999999.999 999999.999 999999.999 999999.999
+
+-- !select --
+2999999.997
+
+-- !select --
+2999999994000.000003
+
+-- !select --
+3.000
+
+-- !select --
+10999999.989
+
diff --git a/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy b/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy
index 301d719b15..284cf482e4 100644
--- a/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy
+++ b/regression-test/suites/datatype_p0/decimalv3/test_arithmetic_expressions.groovy
@@ -49,4 +49,20 @@ suite("test_arithmetic_expressions") {
qt_select "select k1 * k2 * k3 * k1 * k2 * k3 from ${table1} order by k1"
qt_select "select k1 * k2 / k3 * k1 * k2 * k3 from ${table1} order by k1"
sql "drop table if exists ${table1}"
+
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table1} ( `a` DECIMALV3(9, 3) NOT NULL, `b` DECIMALV3(9, 3) NOT NULL, `c` DECIMALV3(9, 3) NOT NULL, `d` DECIMALV3(9, 3) NOT NULL, `e` DECIMALV3(9, 3) NOT NULL, `f` DECIMALV3(9, 3) NOT
+ NULL, `g` DECIMALV3(9, 3) NOT NULL , `h` DECIMALV3(9, 3) NOT NULL, `i` DECIMALV3(9, 3) NOT NULL, `j` DECIMALV3(9, 3) NOT NULL, `k` DECIMALV3(9, 3) NOT NULL) DISTRIBUTED BY HASH(a) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """
+ insert into ${table1} values(999999.999,999999.999,999999.999,999999.999,999999.999,999999.999,999999.999,999999.999,999999.999,999999.999,999999.999);
+ """
+ qt_select_all "select * from ${table1} order by a"
+
+ qt_select "select a + b + c from ${table1};"
+ qt_select "select (a + b + c) * d from ${table1};"
+ qt_select "select (a + b + c) / d from ${table1};"
+ qt_select "select a + b + c + d + e + f + g + h + i + j + k from ${table1};"
+ sql "drop table if exists ${table1}"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 07/07: [Enhancement](spark load)Support for RM HA (#15000)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 50f4a7356426fa2bf9d51d48fca9c4bb7c2ab450
Author: liujinhui <96...@qq.com>
AuthorDate: Tue Mar 7 15:46:14 2023 +0800
[Enhancement](spark load)Support for RM HA (#15000)
Adding RM HA configuration to the spark load.
Spark can accept HA parameters via config, we just need to accept it in the DDL
CREATE EXTERNAL RESOURCE spark_resource_sinan_node_manager_ha
PROPERTIES
(
"type" = "spark",
"spark.master" = "yarn",
"spark.submit.deployMode" = "cluster",
"spark.executor.memory" = "10g",
"spark.yarn.queue" = "XXXX",
"spark.hadoop.yarn.resourcemanager.address" = "XXXX:8032",
"spark.hadoop.yarn.resourcemanager.ha.enabled" = "true",
"spark.hadoop.yarn.resourcemanager.ha.rm-ids" = "rm1,rm2",
"spark.hadoop.yarn.resourcemanager.hostname.rm1" = "XXXX",
"spark.hadoop.yarn.resourcemanager.hostname.rm2" = "XXXX",
"spark.hadoop.fs.defaultFS" = "hdfs://XXXX",
"spark.hadoop.dfs.nameservices" = "hacluster",
"spark.hadoop.dfs.ha.namenodes.hacluster" = "mynamenode1,mynamenode2",
"spark.hadoop.dfs.namenode.rpc-address.hacluster.mynamenode1" = "XXX:8020",
"spark.hadoop.dfs.namenode.rpc-address.hacluster.mynamenode2" = "XXXX:8020",
"spark.hadoop.dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
"working_dir" = "hdfs://XXXX/doris_prd_data/sinan/spark_load/",
"broker" = "broker_personas",
"broker.username" = "hdfs",
"broker.password" = "",
"broker.dfs.nameservices" = "XXX",
"broker.dfs.ha.namenodes.XXX" = "mynamenode1, mynamenode2",
"broker.dfs.namenode.rpc-address.XXXX.mynamenode1" = "XXXX:8020",
"broker.dfs.namenode.rpc-address.XXXX.mynamenode2" = "XXXX:8020",
"broker.dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
);
Co-authored-by: liujh <li...@t3go.cn>
---
.../org/apache/doris/catalog/SparkResource.java | 35 ++++++++++++++++++----
.../apache/doris/catalog/SparkResourceTest.java | 16 ++++++++++
2 files changed, 46 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java
index ac761b5fb6..98ebe54b96 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SparkResource.java
@@ -32,6 +32,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.gson.annotations.SerializedName;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -80,6 +81,10 @@ public class SparkResource extends Resource {
private static final String SPARK_YARN_RESOURCE_MANAGER_ADDRESS = "spark.hadoop.yarn.resourcemanager.address";
private static final String SPARK_FS_DEFAULT_FS = "spark.hadoop.fs.defaultFS";
private static final String YARN_RESOURCE_MANAGER_ADDRESS = "yarn.resourcemanager.address";
+ private static final String SPARK_YARN_RESOURCE_MANAGER_HA_ENABLED = "spark.hadoop.yarn.resourcemanager.ha.enabled";
+ private static final String SPARK_YARN_RESOURCE_MANAGER_HA_RMIDS = "spark.hadoop.yarn.resourcemanager.ha.rm-ids";
+ private static final String YARN_RESOURCE_MANAGER_ADDRESS_FOMART = "spark.hadoop.yarn.resourcemanager.address.%s";
+ private static final String YARN_RESOURCE_MANAGER_HOSTNAME_FORMAT = "spark.hadoop.yarn.resourcemanager.hostname.%s";
public enum DeployMode {
CLUSTER,
@@ -283,11 +288,31 @@ public class SparkResource extends Resource {
throw new DdlException("Missing " + SPARK_SUBMIT_DEPLOY_MODE + " in properties");
}
// if deploy machines do not set HADOOP_CONF_DIR env, we should set these configs blow
- if ((!sparkConfigs.containsKey(SPARK_YARN_RESOURCE_MANAGER_ADDRESS)
- || !sparkConfigs.containsKey(SPARK_FS_DEFAULT_FS))
- && isYarnMaster()) {
- throw new DdlException("Missing (" + SPARK_YARN_RESOURCE_MANAGER_ADDRESS + " and " + SPARK_FS_DEFAULT_FS
- + ") in yarn master");
+ if (isYarnMaster()) {
+ if (!sparkConfigs.containsKey(SPARK_FS_DEFAULT_FS)) {
+ throw new DdlException("Missing (" + SPARK_FS_DEFAULT_FS + ") in yarn master");
+ }
+
+ String haEnabled = sparkConfigs.get(SPARK_YARN_RESOURCE_MANAGER_HA_ENABLED);
+ if (StringUtils.isNotEmpty(haEnabled) && "true".equals(haEnabled)) {
+ if (StringUtils.isEmpty(SPARK_YARN_RESOURCE_MANAGER_HA_RMIDS)) {
+ throw new DdlException("Missing (" + SPARK_YARN_RESOURCE_MANAGER_HA_RMIDS + ") in yarn master, "
+ + "when " + SPARK_YARN_RESOURCE_MANAGER_HA_ENABLED + "=true.");
+ }
+
+ String[] haIds = sparkConfigs.get(SPARK_YARN_RESOURCE_MANAGER_HA_RMIDS).split(",");
+ for (String haId : haIds) {
+ String addressKey = String.format(YARN_RESOURCE_MANAGER_ADDRESS_FOMART, haId);
+ String hostnameKey = String.format(YARN_RESOURCE_MANAGER_HOSTNAME_FORMAT, haId);
+ if (!sparkConfigs.containsKey(addressKey) && !sparkConfigs.containsKey(hostnameKey)) {
+ throw new DdlException("Missing " + addressKey + " or " + hostnameKey + " in yarn master, "
+ + "when " + SPARK_YARN_RESOURCE_MANAGER_HA_ENABLED + "=true.");
+ }
+ }
+ } else if (!sparkConfigs.containsKey(SPARK_YARN_RESOURCE_MANAGER_ADDRESS)) {
+ throw new DdlException("Missing (" + SPARK_YARN_RESOURCE_MANAGER_ADDRESS + ") in yarn master, "
+ + "or not turned on ha.");
+ }
}
// check working dir and broker
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/SparkResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/SparkResourceTest.java
index dc2436c434..0291e122bc 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/SparkResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/SparkResourceTest.java
@@ -117,6 +117,22 @@ public class SparkResourceTest {
BaseProcResult result = new BaseProcResult();
resource.getProcNodeData(result);
Assert.assertEquals(9, result.getRows().size());
+
+ properties.clear();
+ properties.put("type", type);
+ properties.put("spark.master", "yarn");
+ properties.put("spark.submit.deployMode", "cluster");
+ properties.put("spark.hadoop.yarn.resourcemanager.ha.enabled", "true");
+ properties.put("spark.hadoop.yarn.resourcemanager.ha.rm-ids", "rm1,rm2");
+ properties.put("spark.hadoop.yarn.resourcemanager.hostname.rm1", "host1");
+ properties.put("spark.hadoop.yarn.resourcemanager.hostname.rm2", "host2");
+ properties.put("spark.hadoop.fs.defaultFS", "hdfs://127.0.0.1:10000");
+ stmt = new CreateResourceStmt(true, false, name, properties);
+ stmt.analyze(analyzer);
+ resource = (SparkResource) Resource.fromStmt(stmt);
+ Assert.assertTrue(resource.isYarnMaster());
+ map = resource.getSparkConfigs();
+ Assert.assertEquals(7, map.size());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 03/07: [Chore](planner) add error information for OnClause contain ExistsPredicates (#18090)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c4c9e1c1c3663929442ad35033e62907691c62bf
Author: Pxl <px...@qq.com>
AuthorDate: Wed Mar 29 10:47:41 2023 +0800
[Chore](planner) add error information for OnClause contain ExistsPredicates (#18090)
---
fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java | 5 +++++
regression-test/suites/query_p0/join/test_join.groovy | 5 ++++-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
index e7470d1855..e1e274a31d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FromClause.java
@@ -136,6 +136,11 @@ public class FromClause implements ParseNode, Iterable<TableRef> {
}
tblRef.analyze(analyzer);
leftTblRef = tblRef;
+ Expr clause = tblRef.getOnClause();
+ if (clause != null && clause.contains(Subquery.class)) {
+ throw new AnalysisException("Not support OnClause contain Subquery, expr:"
+ + clause.toSql());
+ }
}
// Fix the problem of column nullable attribute error caused by inline view + outer join
changeTblRefToNullable(analyzer);
diff --git a/regression-test/suites/query_p0/join/test_join.groovy b/regression-test/suites/query_p0/join/test_join.groovy
index 56c47e1080..806f56d2d2 100644
--- a/regression-test/suites/query_p0/join/test_join.groovy
+++ b/regression-test/suites/query_p0/join/test_join.groovy
@@ -1098,7 +1098,10 @@ suite("test_join", "query,p0") {
qt_join_on_predicate6"""select count(a.k1) from baseall a join test b on a.k1 < 10 and a.k1 = b.k1"""
qt_join_on_predicate7"""SELECT t2.k1,t2.k2,t3.k1,t3.k2 FROM baseall t2 LEFT JOIN test t3 ON t2.k2=t3.k2 WHERE t2.k1 = 4 OR (t2.k1 > 4 AND t3.k1 IS NULL) order by 1, 2, 3, 4"""
-
+ test {
+ sql "select a.k1 from baseall a join test b on b.k2 in (select 49) and a.k1 = b.k1 order by k1;"
+ exception "Not support OnClause contain Subquery"
+ }
// <=> test cases
qt_join41"""select 1 <=> 2, 1 <=> 1, "a"= \"a\""""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 04/07: [regression-test](log) add log for malforamt response of stream load (#18173)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 069b0a800eb98e4d86d95ed6c540fe85b46c76ea
Author: Mingyu Chen <mo...@163.com>
AuthorDate: Wed Mar 29 15:52:44 2023 +0800
[regression-test](log) add log for malforamt response of stream load (#18173)
---
.../regression/action/StreamLoadAction.groovy | 54 ++++++++++++----------
1 file changed, 29 insertions(+), 25 deletions(-)
diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
index 075f751ae9..18a54d1e15 100644
--- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
+++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/action/StreamLoadAction.groovy
@@ -314,7 +314,6 @@ class StreamLoadAction implements SuiteAction {
long numberLoadedRows = result.NumberLoadedRows.toLong()
if (numberTotalRows != numberLoadedRows) {
throw new IllegalStateException("Stream load rows mismatch:\n${responseText}")
-
}
if (time > 0) {
@@ -333,32 +332,37 @@ class StreamLoadAction implements SuiteAction {
// So here we wait for at most 60s, using "show transaction" to check the
// status of txn, and return once it become ABORTED or VISIBLE.
private String waitForPublishOrFailure(String responseText) {
- long maxWaitSecond = 60;
- def jsonSlurper = new JsonSlurper()
- def parsed = jsonSlurper.parseText(responseText)
- String status = parsed.Status
- long txnId = parsed.TxnId
- if (!status.equalsIgnoreCase("Publish Timeout")) {
- return status;
- }
+ try {
+ long maxWaitSecond = 60;
+ def jsonSlurper = new JsonSlurper()
+ def parsed = jsonSlurper.parseText(responseText)
+ String status = parsed.Status
+ long txnId = parsed.TxnId
+ if (!status.equalsIgnoreCase("Publish Timeout")) {
+ return status;
+ }
- log.info("Stream load with txn ${txnId} is publish timeout")
- String sql = "show transaction from ${db} where id = ${txnId}"
- String st = "PREPARE"
- while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && maxWaitSecond > 0) {
- Thread.sleep(2000)
- maxWaitSecond -= 2
- def (result, meta) = JdbcUtils.executeToStringList(context.getConnection(), sql)
- if (result.size() != 1) {
- throw new IllegalStateException("Failed to get txn's ${txnId}")
+ log.info("Stream load with txn ${txnId} is publish timeout")
+ String sql = "show transaction from ${db} where id = ${txnId}"
+ String st = "PREPARE"
+ while (!st.equalsIgnoreCase("VISIBLE") && !st.equalsIgnoreCase("ABORTED") && maxWaitSecond > 0) {
+ Thread.sleep(2000)
+ maxWaitSecond -= 2
+ def (result, meta) = JdbcUtils.executeToStringList(context.getConnection(), sql)
+ if (result.size() != 1) {
+ throw new IllegalStateException("Failed to get txn's ${txnId}")
+ }
+ st = String.valueOf(result[0][3])
}
- st = String.valueOf(result[0][3])
- }
- log.info("Stream load with txn ${txnId} is ${st}")
- if (st.equalsIgnoreCase("VISIBLE")) {
- return "Success";
- } else {
- return "Fail";
+ log.info("Stream load with txn ${txnId} is ${st}")
+ if (st.equalsIgnoreCase("VISIBLE")) {
+ return "Success";
+ } else {
+ return "Fail";
+ }
+ } catch (Throwable t) {
+ log.info("failed to waitForPublishOrFailure. response: ${responseText}", t);
+ throw t;
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 06/07: [Fix](planner)fix create view ignore order by info bug. (#18197)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit ad052f8e4794467c479ab46536f3c6f96ce70396
Author: mch_ucchi <41...@users.noreply.github.com>
AuthorDate: Thu Mar 30 20:17:46 2023 +0800
[Fix](planner)fix create view ignore order by info bug. (#18197)
---
.../java/org/apache/doris/analysis/SelectStmt.java | 9 +-
.../org/apache/doris/catalog/CreateViewTest.java | 2 +-
.../suites/query_p0/view/test_view.groovy | 104 +++++++++++++++++++++
3 files changed, 107 insertions(+), 8 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 3c8e288fb0..5884ea7d0f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -55,6 +55,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -2011,13 +2012,7 @@ public class SelectStmt extends QueryStmt {
// Order By clause
if (orderByElements != null) {
strBuilder.append(" ORDER BY ");
- for (int i = 0; i < orderByElements.size(); ++i) {
- strBuilder.append(orderByElements.get(i).getExpr().toSql());
- if (sortInfo != null) {
- strBuilder.append((sortInfo.getIsAscOrder().get(i)) ? " ASC" : " DESC");
- }
- strBuilder.append((i + 1 != orderByElements.size()) ? ", " : "");
- }
+ strBuilder.append(StringUtils.join(orderByElements, ", "));
}
// Limit clause.
if (hasLimitClause()) {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java
index 1f2e98ea7c..0c9d26fd64 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java
@@ -191,7 +191,7 @@ public class CreateViewTest {
Assert.assertEquals(
"WITH test1_cte(w1, w2) AS (SELECT `k1`, `k2` FROM `default_cluster:test`.`tbl1`) "
+ "SELECT `w1` AS `c1`, sum(`w2`) AS `c2` FROM `test1_cte` WHERE `w1` > 10 GROUP BY `w1` "
- + "ORDER BY `w1` ASC",
+ + "ORDER BY `w1` ASC NULLS FIRST",
alter1.getInlineViewDef());
}
}
diff --git a/regression-test/suites/query_p0/view/test_view.groovy b/regression-test/suites/query_p0/view/test_view.groovy
new file mode 100644
index 0000000000..16867889e3
--- /dev/null
+++ b/regression-test/suites/query_p0/view/test_view.groovy
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_view") {
+ sql "use test_query_db"
+ sql "drop table if exists test_insert"
+ sql '''
+ CREATE TABLE `test_insert` (
+ `id` varchar(11) NULL,
+ `name` varchar(10) NULL,
+ `age` int(11) NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`id`)
+ COMMENT 'test\'
+ DISTRIBUTED BY HASH(`id`) BUCKETS 10
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "in_memory" = "true",
+ "storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false"
+ );
+ '''
+
+ sql '''
+ INSERT INTO `test_insert` (`id`, `name`, `age`) VALUES
+ ('10006', 'doris', 10006), ('10025', 'doris', 10025), ('10031', 'doris', 10031), ('10046', 'doris', 10046),
+ ('10051', 'doris', 10051), ('10070', 'doris', 10070), ('10072', 'doris', 10072), ('10079', 'doris', 10079),
+ ('10085', 'doris', 10085), ('10086', 'doris', 10086), ('10099', 'doris', 10099), ('10101', 'doris', 10101),
+ ('10115', 'doris', 10115), ('10140', 'doris', 10140), ('10142', 'doris', 10142), ('10149', 'doris', 10149),
+ ('10155', 'doris', 10155), ('10156', 'doris', 10156), ('10157', 'doris', 10157), ('10176', 'doris', 10176),
+ ('10183', 'doris', 10183), ('10196', 'doris', 10196), ('10204', 'doris', 10204), ('10233', 'doris', 10233),
+ ('10238', 'doris', 10238), ('10247', 'doris', 10247), ('10264', 'doris', 10264), ('10265', 'doris', 10265),
+ ('1027', 'doris', 1027), ('10270', 'doris', 10270), ('10271', 'doris', 10271), ('10302', 'doris', 10302),
+ ('10314', 'doris', 10314), ('10316', 'doris', 10316), ('10320', 'doris', 10320), ('10321', 'doris', 10321),
+ ('10328', 'doris', 10328), ('10343', 'doris', 10343), ('10360', 'doris', 10360), ('10374', 'doris', 10374),
+ ('10377', 'doris', 10377), ('1038', 'doris', 1038), ('10381', 'doris', 10381), ('10382', 'doris', 10382),
+ ('10407', 'doris', 10407), ('10411', 'doris', 10411), ('10424', 'doris', 10424), ('10426', 'doris', 10426),
+ ('10427', 'doris', 10427), ('10431', 'doris', 10431), ('10459', 'doris', 10459), ('10466', 'doris', 10466),
+ ('1047', 'doris', 1047), ('10534', 'doris', 10534), ('10582', 'doris', 10582), ('10589', 'doris', 10589),
+ ('10594', 'doris', 10594), ('10597', 'doris', 10597), ('10605', 'doris', 10605), ('10606', 'doris', 10606),
+ ('10607', 'doris', 10607), ('10619', 'doris', 10619), ('10625', 'doris', 10625), ('10631', 'doris', 10631),
+ ('10633', 'doris', 10633), ('10638', 'doris', 10638), ('1065', 'doris', 1065), ('10651', 'doris', 10651),
+ ('1067', 'doris', 1067), ('10678', 'doris', 10678), ('10686', 'doris', 10686), ('10687', 'doris', 10687),
+ ('10692', 'doris', 10692), ('10699', 'doris', 10699), ('1070', 'doris', 1070), ('10709', 'doris', 10709),
+ ('10716', 'doris', 10716), ('10728', 'doris', 10728), ('10729', 'doris', 10729), ('10741', 'doris', 10741),
+ ('10743', 'doris', 10743), ('10760', 'doris', 10760), ('10776', 'doris', 10776), ('1078', 'doris', 1078),
+ ('10795', 'doris', 10795), ('108', 'doris', 108), ('10811', 'doris', 10811), ('10844', 'doris', 10844),
+ ('10846', 'doris', 10846), ('10864', 'doris', 10864), ('10873', 'doris', 10873), ('10879', 'doris', 10879),
+ ('10886', 'doris', 10886), ('10899', 'doris', 10899), ('10908', 'doris', 10908), ('1091', 'doris', 1091),
+ ('10914', 'doris', 10914), ('10917', 'doris', 10917), ('1092', 'doris', 1092), ('1093', 'doris', 1093),
+ ('10934', 'doris', 10934), ('10937', 'doris', 10937), ('10942', 'doris', 10942), ('10948', 'doris', 10948),
+ ('10954', 'doris', 10954), ('10955', 'doris', 10955), ('10957', 'doris', 10957), ('10961', 'doris', 10961),
+ ('10968', 'doris', 10968), ('10976', 'doris', 10976), ('10977', 'doris', 10977), ('10980', 'doris', 10980),
+ ('10981', 'doris', 10981), ('10988', 'doris', 10988), ('10994', 'doris', 10994), ('10997', 'doris', 10997),
+ ('11003', 'doris', 11003), ('11016', 'doris', 11016), ('11020', 'doris', 11020), ('11028', 'doris', 11028),
+ ('11029', 'doris', 11029), ('11037', 'doris', 11037), ('11041', 'doris', 11041), ('11057', 'doris', 11057),
+ ('11081', 'doris', 11081), ('11096', 'doris', 11096), ('11125', 'doris', 11125), ('11139', 'doris', 11139),
+ ('11150', 'doris', 11150), ('11164', 'doris', 11164), ('11165', 'doris', 11165), ('11167', 'doris', 11167),
+ ('11172', 'doris', 11172), ('11185', 'doris', 11185), ('11192', 'doris', 11192), ('1120', 'doris', 1120),
+ ('11203', 'doris', 11203), ('1121', 'doris', 1121), ('11215', 'doris', 11215), ('11217', 'doris', 11217),
+ ('11221', 'doris', 11221), ('11229', 'doris', 11229), ('11243', 'doris', 11243), ('11255', 'doris', 11255),
+ ('1128', 'doris', 1128), ('11281', 'doris', 11281), ('11283', 'doris', 11283), ('11288', 'doris', 11288),
+ ('1129', 'doris', 1129), ('11296', 'doris', 11296), ('11305', 'doris', 11305), ('11307', 'doris', 11307),
+ ('11332', 'doris', 11332), ('11347', 'doris', 11347), ('11351', 'doris', 11351), ('11358', 'doris', 11358),
+ ('11366', 'doris', 11366), ('11373', 'doris', 11373), ('11379', 'doris', 11379), ('11385', 'doris', 11385),
+ ('11390', 'doris', 11390), ('11392', 'doris', 11392), ('11399', 'doris', 11399), ('1140', 'doris', 1140),
+ ('11400', 'doris', 11400), ('11402', 'doris', 11402), ('11417', 'doris', 11417), ('11436', 'doris', 11436),
+ ('11448', 'doris', 11448), ('11455', 'doris', 11455), ('11460', 'doris', 11460), ('11468', 'doris', 11468),
+ ('11469', 'doris', 11469), ('11475', 'doris', 11475), ('1148', 'doris', 1148), ('11488', 'doris', 11488),
+ ('11506', 'doris', 11506), ('11518', 'doris', 11518), ('11531', 'doris', 11531), ('11532', 'doris', 11532),
+ ('11550', 'doris', 11550), ('11551', 'doris', 11551), ('11592', 'doris', 11592), ('1160', 'doris', 1160),
+ ('11603', 'doris', 11603), ('1161', 'doris', 1161), ('11615', 'doris', 11615), ('11617', 'doris', 11617),
+ ('11623', 'doris', 11623), ('11629', 'doris', 11629), ('11634', 'doris', 11634), ('11637', 'doris', 11637),
+ ('11642', 'doris', 11642), ('11656', 'doris', 11656), ('11663', 'doris', 11663), ('11688', 'doris', 11688),
+ ('11689', 'doris', 11689), ('1169', 'doris', 1169), ('117', 'doris', 117), ('11710', 'doris', 11710),
+ ('11711', 'doris', 11711)
+ '''
+
+ sql "drop view if exists v"
+ sql "CREATE VIEW v (id, name, age) AS SELECT id, name, age FROM test_insert order by age desc limit 2;"
+
+ test {
+ sql "select * from v"
+ result([
+ ['11711', 'doris', 11711],
+ ['11710', 'doris', 11710]
+ ])
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 05/07: [Fix](planner) fix nested udf bind arguments exception (#18188)
Posted by mo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 8f8adea52bf2d7d7ea3d02b5d509945458261be5
Author: mch_ucchi <41...@users.noreply.github.com>
AuthorDate: Thu Mar 30 11:39:02 2023 +0800
[Fix](planner) fix nested udf bind arguments exception (#18188)
nested alias function will cause bind argument exception, sql like:
``` sql
CREATE ALIAS FUNCTION f1(DATETIMEV2(3), INT)
with PARAMETER (datetime1, int1) as date_trunc(days_sub(datetime1, int1), 'day')
CREATE ALIAS FUNCTION f2(DATETIMEV2(3), int)
with PARAMETER (datetime1, int1) as DATE_FORMAT(HOURS_ADD(
date_trunc(datetime1, 'day'),
add(multiply(floor(divide(HOUR(datetime1), divide(24,int1))), 1), 1)
), '%Y%m%d:%H')
select f2(f1(now(3), 2), 3)
```
bug in FunctionCallExpr#rewriteExpr(), the retExpr will be replaced to originExpr to change the alias function to builtin function, but the retExpr.fn is not null, so when return to outer scope, the fn will be covered. That's the example:
```
f1(f1()) -> date_trunc(days_sub(date_trunc(days_sub()))) is correct and
f1(f1()) -> date_trunc(days_sub(days_sub())) is bug.
```
we fix it.
---
.../apache/doris/analysis/FunctionCallExpr.java | 3 ++
.../sql_functions/test_alias_function.groovy | 34 ++++++++++++++++++++++
2 files changed, 37 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index a91a01c4a6..e562706073 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1583,6 +1583,9 @@ public class FunctionCallExpr extends Expr {
retExpr.fnParams = new FunctionParams(oriExpr.fnParams.isDistinct(), oriParamsExprs);
+ // retExpr changed to original function, so the fn should be null.
+ retExpr.fn = null;
+
// reset children
retExpr.children.clear();
retExpr.children.addAll(oriExpr.getChildren());
diff --git a/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy b/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
new file mode 100644
index 0000000000..4693c79067
--- /dev/null
+++ b/regression-test/suites/query_p0/sql_functions/test_alias_function.groovy
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite('test_alias_function') {
+ sql "use test_query_db"
+ sql '''
+ CREATE ALIAS FUNCTION IF NOT EXISTS f1(DATETIMEV2(3), INT)
+ with PARAMETER (datetime1, int1) as date_trunc(days_sub(datetime1, int1), 'day')'''
+ sql '''
+ CREATE ALIAS FUNCTION IF NOT EXISTS f2(DATETIMEV2(3), int)
+ with PARAMETER (datetime1, int1) as DATE_FORMAT(HOURS_ADD(
+ date_trunc(datetime1, 'day'),
+ add(multiply(floor(divide(HOUR(datetime1), divide(24,int1))), 1), 1)
+ ), '%Y%m%d:%H');'''
+
+ test {
+ sql 'select f2(f1(now(3), 2), 3)'
+ result([['20230327:01']])
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org