You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by da...@apache.org on 2024/07/16 03:46:27 UTC
(doris) branch branch-3.0 updated (13a42f142c2 -> d643e85a7fb)
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a change to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 13a42f142c2 [opt](test) move test_s3_tvf case to p2 (#37624)
new 5e35d950b71 [test](fix) replace hardcode s3BucketName (#37739)
new d643e85a7fb [fix](nerieds) set row count to colStats if colStats.isUnknown (#37522)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../doris/nereids/stats/StatsCalculator.java | 6 +
.../hive/test_hive_write_insert_s3.groovy | 5 +-
.../paimon/paimon_base_filesystem.groovy | 5 +-
.../suites/github_events_p2/load.groovy | 2 +-
.../load_p0/broker_load/test_compress_type.groovy | 41 ++---
.../test_csv_with_enclose_and_escapeS3_load.groovy | 15 +-
.../load_p0/broker_load/test_etl_failed.groovy | 8 +-
.../broker_load/test_multi_table_load.groovy | 15 +-
.../suites/load_p0/broker_load/test_s3_load.groovy | 44 ++---
.../load_p0/broker_load/test_seq_load.groovy | 7 +-
.../load_p2/broker_load/test_broker_load.groovy | 140 +++++++--------
.../test_parquet_large_metadata_load.groovy | 15 +-
.../broker_load/test_s3_load_properties.groovy | 150 ++++++++--------
.../test_s3_load_with_load_parallelism.groovy | 7 +-
.../tvf/test_tvf_based_broker_load.groovy | 74 ++++----
.../suites/load_p2/tvf/test_s3_tvf.groovy | 190 +++++++++++----------
.../diff_data/stress_test_diff_date_list.groovy | 2 +-
.../same_data/stress_test_same_date_range.groovy | 2 +-
.../stress_test_two_stream_load.groovy | 2 +-
.../query_profile/s3_load_profile_test.groovy | 8 +-
...est_update_rows_and_partition_first_load.groovy | 16 +-
regression-test/suites/tpcds_sf1000_p2/load.groovy | 5 +-
22 files changed, 395 insertions(+), 364 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 01/02: [test](fix) replace hardcode s3BucketName (#37739)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5e35d950b7144243d90bdfe2564db9a4b2bad61b
Author: Dongyang Li <he...@qq.com>
AuthorDate: Mon Jul 15 20:23:37 2024 +0800
[test](fix) replace hardcode s3BucketName (#37739)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---------
Co-authored-by: stephen <he...@qq.com>
---
.../hive/test_hive_write_insert_s3.groovy | 5 +-
.../paimon/paimon_base_filesystem.groovy | 5 +-
.../suites/github_events_p2/load.groovy | 2 +-
.../load_p0/broker_load/test_compress_type.groovy | 41 ++---
.../test_csv_with_enclose_and_escapeS3_load.groovy | 15 +-
.../load_p0/broker_load/test_etl_failed.groovy | 8 +-
.../broker_load/test_multi_table_load.groovy | 15 +-
.../suites/load_p0/broker_load/test_s3_load.groovy | 44 ++---
.../load_p0/broker_load/test_seq_load.groovy | 7 +-
.../load_p2/broker_load/test_broker_load.groovy | 140 +++++++--------
.../test_parquet_large_metadata_load.groovy | 15 +-
.../broker_load/test_s3_load_properties.groovy | 150 ++++++++--------
.../test_s3_load_with_load_parallelism.groovy | 7 +-
.../tvf/test_tvf_based_broker_load.groovy | 74 ++++----
.../suites/load_p2/tvf/test_s3_tvf.groovy | 190 +++++++++++----------
.../diff_data/stress_test_diff_date_list.groovy | 2 +-
.../same_data/stress_test_same_date_range.groovy | 2 +-
.../stress_test_two_stream_load.groovy | 2 +-
.../query_profile/s3_load_profile_test.groovy | 8 +-
...est_update_rows_and_partition_first_load.groovy | 16 +-
regression-test/suites/tpcds_sf1000_p2/load.groovy | 5 +-
21 files changed, 389 insertions(+), 364 deletions(-)
diff --git a/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy b/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy
index 87633ba1b09..cf9fea67cbd 100644
--- a/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy
+++ b/regression-test/suites/external_table_p2/hive/test_hive_write_insert_s3.groovy
@@ -17,6 +17,7 @@
suite("test_hive_write_insert_s3", "p2,external,hive,external_remote,external_remote_hive") {
def format_compressions = ["parquet_snappy"]
+ def s3BucketName = getS3BucketName()
def q01 = { String format_compression, String catalog_name ->
logger.info("hive sql: " + """ truncate table all_types_${format_compression}_s3; """)
@@ -76,8 +77,8 @@ suite("test_hive_write_insert_s3", "p2,external,hive,external_remote,external_re
hive_remote """ DROP TABLE IF EXISTS all_types_par_${format_compression}_s3_${catalog_name}_q02; """
logger.info("hive sql: " + """ CREATE TABLE IF NOT EXISTS all_types_par_${format_compression}_s3_${catalog_name}_q02 like all_types_par_${format_compression}_s3; """)
hive_remote """ CREATE TABLE IF NOT EXISTS all_types_par_${format_compression}_s3_${catalog_name}_q02 like all_types_par_${format_compression}_s3; """
- logger.info("hive sql: " + """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://doris-build-1308700295/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """)
- hive_remote """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://doris-build-1308700295/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """
+ logger.info("hive sql: " + """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://${s3BucketName}/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """)
+ hive_remote """ ALTER TABLE all_types_par_${format_compression}_s3_${catalog_name}_q02 SET LOCATION 'cosn://${s3BucketName}/regression/write/data/all_types_par_${format_compression}_s3_${catalog_name}_q02'; """
sql """refresh catalog ${catalog_name};"""
sql """
diff --git a/regression-test/suites/external_table_p2/paimon/paimon_base_filesystem.groovy b/regression-test/suites/external_table_p2/paimon/paimon_base_filesystem.groovy
index a091e3615fa..b305e609604 100644
--- a/regression-test/suites/external_table_p2/paimon/paimon_base_filesystem.groovy
+++ b/regression-test/suites/external_table_p2/paimon/paimon_base_filesystem.groovy
@@ -25,6 +25,7 @@ suite("paimon_base_filesystem", "p2,external,paimon,external_remote,external_rem
String s3ak = getS3AK()
String s3sk = getS3SK()
+ def s3Endpoint = getS3Endpoint()
def cos = """select c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c18 from ${catalog_cos}.zd.all_table order by c18"""
def oss = """select * from ${catalog_oss}.paimonossdb1.test_tableoss order by a"""
@@ -44,9 +45,9 @@ suite("paimon_base_filesystem", "p2,external,paimon,external_remote,external_rem
create catalog if not exists ${catalog_oss} properties (
"type" = "paimon",
"warehouse" = "oss://paimon-zd/paimonoss",
- "oss.endpoint"="oss-cn-beijing.aliyuncs.com",
"oss.access_key"="${ak}",
- "oss.secret_key"="${sk}"
+ "oss.secret_key"="${sk}",
+ "oss.endpoint"="oss-cn-beijing.aliyuncs.com"
);
"""
logger.info("catalog " + catalog_cos + " created")
diff --git a/regression-test/suites/github_events_p2/load.groovy b/regression-test/suites/github_events_p2/load.groovy
index dc2e0dbb975..92a588a2214 100644
--- a/regression-test/suites/github_events_p2/load.groovy
+++ b/regression-test/suites/github_events_p2/load.groovy
@@ -31,7 +31,7 @@ suite("load") {
ak "${getS3AK()}"
sk "${getS3SK()}"
endpoint "http://${getS3Endpoint()}"
- region "ap-beijing"
+ region "${getS3Region()}"
repository "regression_test_github_events"
snapshot "github_events"
timestamp "2022-03-23-12-19-51"
diff --git a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy
index 693e533fa50..723a07d5296 100644
--- a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy
@@ -17,6 +17,7 @@
suite("test_compress_type", "load_p0") {
def tableName = "basic_data"
+ def s3BucketName = getS3BucketName()
// GZ/LZO/BZ2/LZ4FRAME/DEFLATE/LZOP
def compressTypes = [
@@ -62,24 +63,24 @@ suite("test_compress_type", "load_p0") {
]
def paths = [
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
- "s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
- "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.gz",
- "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.bz2",
- "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.lz4",
- "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.gz",
- "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.bz2",
- "s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.lz4",
+ "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.gz",
+ "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.bz2",
+ "s3://${s3BucketName}/regression/load/data/basic_data_by_line.json.lz4",
]
def labels = []
@@ -137,8 +138,8 @@ suite("test_compress_type", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${getS3Endpoint()}",
+ "AWS_REGION" = "${getS3Region()}",
"provider" = "${getS3Provider()}"
)
"""
diff --git a/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy b/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy
index eea25fb4534..291f623a512 100644
--- a/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_csv_with_enclose_and_escapeS3_load.groovy
@@ -19,6 +19,7 @@
suite("test_csv_with_enclose_and_escapeS3_load", "load_p0") {
def tableName = "test_csv_with_enclose_and_escape"
+ def s3BucketName = getS3BucketName()
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
@@ -48,24 +49,24 @@ suite("test_csv_with_enclose_and_escapeS3_load", "load_p0") {
]
for (i in 0..<normalCases.size()) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/${normalCases[i]}.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/${normalCases[i]}.csv",
"${tableName}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"CSV\"", "(k1,k2,v1,v2,v3,v4)",
"PROPERTIES (\"enclose\" = \"\\\"\", \"escape\" = \"\\\\\", \"trim_double_quotes\" = \"true\")"))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/enclose_incomplete.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/enclose_incomplete.csv",
"${tableName}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"CSV\"", "(k1,k2,v1,v2,v3,v4)",
"PROPERTIES (\"enclose\" = \"\\\"\", \"escape\" = \"\\\\\", \"trim_double_quotes\" = \"true\")").addProperties("max_filter_ratio", "0.5"))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/enclose_without_escape.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/enclose_without_escape.csv",
"${tableName}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"CSV\"", "(k1,k2,v1,v2,v3,v4)",
"PROPERTIES (\"enclose\" = \"\\\"\", \"escape\" = \"\\\\\", \"trim_double_quotes\" = \"true\")"))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/enclose_multi_char_delimiter.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/enclose_multi_char_delimiter.csv",
"${tableName}", "LINES TERMINATED BY \"\$\$\$\"", "COLUMNS TERMINATED BY \"@@\"", "FORMAT AS \"CSV\"", "(k1,k2,v1,v2,v3,v4)",
"PROPERTIES (\"enclose\" = \"\\\"\", \"escape\" = \"\\\\\", \"trim_double_quotes\" = \"true\")"))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/enclose_not_trim_quotes.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/enclose_not_trim_quotes.csv",
"${tableName}", "", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"CSV\"", "(k1,k2,v1,v2,v3,v4)",
"PROPERTIES (\"enclose\" = \"\\\"\", \"escape\" = \"\\\\\")").addProperties("trim_double_quotes", "false"))
@@ -92,8 +93,8 @@ suite("test_csv_with_enclose_and_escapeS3_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${getS3Endpoint()}",
+ "AWS_REGION" = "${getS3Region()}",
"provider" = "${getS3Provider()}"
)
${prop}
diff --git a/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy b/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
index 184d68ed08c..30da06c2c98 100644
--- a/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_etl_failed.groovy
@@ -16,6 +16,8 @@
// under the License.
suite("test_etl_failed", "load_p0") {
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
def tableName = "test_etl_failed"
sql """ DROP TABLE IF EXISTS ${tableName} """
sql """
@@ -33,7 +35,7 @@ suite("test_etl_failed", "load_p0") {
PROPERTIES ("replication_allocation" = "tag.location.default: 1");
"""
String label = "test_etl_failed"
- String path = "s3://doris-build-1308700295/regression/load/data/etl_failure/etl-failure.csv"
+ String path = "s3://${s3BucketName}/regression/load/data/etl_failure/etl-failure.csv"
String format = "CSV"
String ak = getS3AK()
String sk = getS3SK()
@@ -46,8 +48,8 @@ suite("test_etl_failed", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
PROPERTIES(
diff --git a/regression-test/suites/load_p0/broker_load/test_multi_table_load.groovy b/regression-test/suites/load_p0/broker_load/test_multi_table_load.groovy
index 31f07952674..d9dc1da355d 100644
--- a/regression-test/suites/load_p0/broker_load/test_multi_table_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_multi_table_load.groovy
@@ -16,7 +16,10 @@
// under the License.
suite("test_multi_table_load", "load_p0") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
+
def tableName = "test_multi_table_load"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -81,7 +84,7 @@ suite("test_multi_table_load", "load_p0") {
);
"""
- def path = "s3://doris-build-1308700295/regression/load/data/basic_data.csv"
+ def path = "s3://${s3BucketName}/regression/load/data/basic_data.csv"
def format_str = "CSV"
def ak = getS3AK()
def sk = getS3SK()
@@ -129,8 +132,8 @@ suite("test_multi_table_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
properties(
@@ -178,8 +181,8 @@ suite("test_multi_table_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
properties(
diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
index f84a44959aa..7e57af5aabb 100644
--- a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_s3_load", "load_p0") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
sql "create workload group if not exists broker_load_test properties ( 'cpu_share'='1024'); "
sql "set workload_group=broker_load_test;"
@@ -98,71 +100,71 @@ suite("test_s3_load", "load_p0") {
]
/* ========================================================== normal ========================================================== */
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "", "", ""))
/* ========================================================== error ========================================================== */
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
/* ========================================================== wrong column sep ========================================================== */
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
/* ========================================================== wrong line delim ========================================================== */
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
/* ========================================================== strict mode ========================================================== */
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "","", true).addProperties("strict_mode", "true"))
/* ========================================================== timezone ========================================================== */
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "Asia/Shanghai"))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "America/Chicago"))
/* ========================================================== compress type ========================================================== */
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
@@ -172,20 +174,20 @@ suite("test_s3_load", "load_p0") {
/*========================================================== json ==========================================================*/
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.json",
"${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")"))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_by_line.json",
"${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.parq",
"${table}", "", "", "FORMAT AS \"parquet\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "", "", ""))
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.orc",
"${table}", "", "", "FORMAT AS \"orc\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
@@ -217,8 +219,8 @@ suite("test_s3_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"use_path_style" = "$attributes.usePathStyle",
"provider" = "${getS3Provider()}"
)
diff --git a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
index 0c94467c6a5..da91277b3fa 100644
--- a/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_seq_load.groovy
@@ -16,7 +16,6 @@
// under the License.
suite("test_seq_load", "load_p0") {
-
def tableName = "uniq_tbl_basic_seq"
sql """ DROP TABLE IF EXISTS ${tableName} """
@@ -83,7 +82,7 @@ suite("test_seq_load", "load_p0") {
"""
def label = UUID.randomUUID().toString().replace("-", "0")
- def path = "s3://doris-build-1308700295/regression/load/data/basic_data.csv"
+ def path = "s3://${getS3BucketName()}/regression/load/data/basic_data.csv"
def format_str = "CSV"
def ak = getS3AK()
def sk = getS3SK()
@@ -101,8 +100,8 @@ suite("test_seq_load", "load_p0") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing"
+ "AWS_ENDPOINT" = "${getS3Endpoint()}",
+ "AWS_REGION" = "${getS3Region()}"
)
properties(
"use_new_load_scan_node" = "true"
diff --git a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
index 5b88e4087e1..e5f8272f986 100644
--- a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
+++ b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_broker_load_p2", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = ["part",
"upper_case",
"reverse",
@@ -49,37 +51,37 @@ suite("test_broker_load_p2", "p2") {
"orc_s3_case9", // table column uppercase * load column lowercase * orc file uppercase
"csv_s3_case_line_delimiter" // csv format table with special line delimiter
]
- def paths = ["s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/path/*/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_100k_rows.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/line_delimiter/lineitem_0x7.csv.gz"
+ def paths = ["s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/path/*/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz"
]
def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
"""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
@@ -194,37 +196,37 @@ suite("test_broker_load_p2", "p2") {
"\\N"
]
- def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0",
- "cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0"
+ def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0",
+ "cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"
]
def error_msg = ["",
@@ -291,8 +293,8 @@ suite("test_broker_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
properties(
@@ -392,11 +394,11 @@ suite("test_broker_load_p2", "p2") {
def label_22666 = "part_" + UUID.randomUUID().toString().replace("-", "0")
sql """
LOAD LABEL ${label_22666} (
- DATA INFILE("s3://doris-build-1308700295/regression/load/data/part0.parquet")
+ DATA INFILE("s3://${s3BucketName}/regression/load/data/part0.parquet")
INTO TABLE ${tbl_22666}
FORMAT AS "PARQUET"
(p_partkey, p_name, p_mfgr),
- DATA INFILE("s3://doris-build-1308700295/regression/load/data/part1.parquet")
+ DATA INFILE("s3://${s3BucketName}/regression/load/data/part1.parquet")
INTO TABLE ${tbl_22666}
FORMAT AS "PARQUET"
(p_partkey, p_brand, p_type)
@@ -404,8 +406,8 @@ suite("test_broker_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
);
"""
diff --git a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
index 0712f8cead3..64c7c3c0438 100644
--- a/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
+++ b/regression-test/suites/load_p2/broker_load/test_parquet_large_metadata_load.groovy
@@ -16,10 +16,11 @@
// under the License.
suite("test_parquet_large_metadata_load_p2", "p2") {
-
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = ["parquet_large_metadata_100mb" // metadata size more than 100MB
]
- def paths = ["s3://doris-build-1308700295/regression/load/metadata/parquet_large_metadata_100mb.parquet"
+ def paths = ["s3://${getS3BucketName()}/regression/load/metadata/parquet_large_metadata_100mb.parquet"
]
String ak = getS3AK()
String sk = getS3SK()
@@ -27,10 +28,10 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
def expect_tvf_result = """[[2, 8], [2, 8], [2, 8], [2, 8], [2, 8]]"""
String[][] tvf_result = sql """select `1`,`2` from s3(
- "uri" = "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/metadata/parquet_large_metadata_100mb.parquet",
+ "uri" = "https://${getS3BucketName()}.${getS3Endpoint()}/regression/load/metadata/parquet_large_metadata_100mb.parquet",
"s3.access_key" = "$ak",
"s3.secret_key" = "$sk",
- "s3.region" = "ap-beijing",
+ "s3.region" = "${s3Region}",
"provider" = "${getS3Provider()}",
"format" = "parquet"
) order by `1`,`2` limit 5;
@@ -48,8 +49,8 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
PROPERTIES
@@ -61,7 +62,7 @@ suite("test_parquet_large_metadata_load_p2", "p2") {
}
def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=45000"]
- def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0"]
+ def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"]
def error_msg = [""]
// test unified load
if (enabled != null && enabled.equalsIgnoreCase("true")) {
diff --git a/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy b/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy
index b837f28f171..fbb0df3b002 100644
--- a/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy
+++ b/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_s3_load_properties", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
sql "create workload group if not exists broker_load_test properties ( 'cpu_share'='1024'); "
sql "set workload_group=broker_load_test;"
@@ -55,183 +57,183 @@ suite("test_s3_load_properties", "p2") {
/* ========================================================== normal ========================================================== */
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(K00,K01,K02,K03,K04,K05,K06,k07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "SET (K19=to_bitmap(k04),K20=HLL_HASH(k04),K21=TO_QUANTILE_STATE(K04,1.0),Kd19=to_bitmap(K05),kd20=HLL_HASH(K05),KD21=TO_QUANTILE_STATE(K05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17)",
"", "", "", "", ""))
}
// TODO: should be success ?
// for (String table : basicTables) {
-// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "", "").withPathStyle())
// }
//
-// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").withPathStyle())
//
// for (String table : arrayTables) {
-// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv",
+// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_array_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", "").withPathStyle())
// }
//
// for (String table : basicTables) {
-// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "", "").withPathStyle())
// }
//
-// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv",
+// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").withPathStyle())
//
// for (String table : arrayTables) {
-// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv",
+// attributesList.add(new LoadAttributes("s3://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_array_data.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", "").withPathStyle())
// }
/* ========================================================== error ========================================================== */
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "", true))
}
// has problem, should be success
// for(String table: basicTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "","").addProperties("max_filter_ratio", "0.5"))
// }
//
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("max_filter_ratio", "0.5"))
//
// for(String table : arrayTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "","").addProperties("max_filter_ratio", "0.5"))
// }
// for(String table: basicTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "","", true).addProperties("max_filter_ratio", "0.4"))
// }
//
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true).addProperties("max_filter_ratio", "0.4"))
//
// for(String table : arrayTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "","", true).addProperties("max_filter_ratio", "0.4"))
// }
// skip lines
// for(String table: basicTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "", "","").addProperties("skip_lines", "10"))
// }
//
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("skip_lines", "10"))
//
// for(String table : arrayTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "","").addProperties("skip_lines", "10"))
// }
/* ========================================================== wrong column sep ========================================================== */
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "", true))
}
/* ========================================================== wrong line delim ========================================================== */
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "", true))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "", true))
}
/* ========================================================== strict mode ========================================================== */
for(String table: basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "","", true).addProperties("strict_mode", "true"))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_with_errors.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true).addProperties("strict_mode","true"))
for(String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_with_errors.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "","", true).addProperties("strict_mode", "true"))
}
@@ -239,131 +241,131 @@ suite("test_s3_load_properties", "p2") {
/* ========================================================== timezone ========================================================== */
for(String table: basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "Asia/Shanghai"))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("timezone", "Asia/Shanghai"))
for(String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "Asia/Shanghai"))
}
for(String table: basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "America/Chicago"))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("timezone", "America/Chicago"))
for(String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "America/Chicago"))
}
/* ========================================================== compress type ========================================================== */
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.gz",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.gz",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.bz2",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.bz2",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv.lz4",
"agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.csv.lz4",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", ""))
}
@@ -371,7 +373,7 @@ suite("test_s3_load_properties", "p2") {
/*========================================================== order by ==========================================================*/
for (String table : uniqTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "ORDER BY k01"))
}
@@ -379,71 +381,71 @@ suite("test_s3_load_properties", "p2") {
/*========================================================== json ==========================================================*/
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.json",
"${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")"))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.json",
"agg_tbl_basic", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")"))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.json",
"${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")"))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_by_line.json",
"${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_by_line.json",
"agg_tbl_basic", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
for (String table : arrayTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_by_line.json",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data_by_line.json",
"${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
"", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
}
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.parq",
"${table}", "", "", "FORMAT AS \"parquet\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.parq",
"agg_tbl_basic", "", "", "FORMAT AS \"PARQUET\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
// for (String table : arrayTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.parq",
// "${table}", "", "", "FORMAT AS \"parquet\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", ""))
// }
for (String table : basicTables) {
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.orc",
"${table}", "", "", "FORMAT AS \"orc\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", ""))
}
- attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
+ attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data.orc",
"agg_tbl_basic", "", "", "FORMAT AS \"ORC\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", ""))
// for (String table : arrayTables) {
-// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
+// attributesList.add(new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_array_data.parq",
// "${table}", "", "", "FORMAT AS \"parquet\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
// "", "", "", "", ""))
// }
for(String table : uniqTables) {
- def attributes = new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_delete.csv",
+ def attributes = new LoadAttributes("s3://${s3BucketName}/regression/load/data/basic_data_delete.csv",
"${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,__DEL__)",
"", "", "", "", "DELETE ON __DEL__=true")
attributes.dataDesc.mergeType = "MERGE"
@@ -482,8 +484,8 @@ suite("test_s3_load_properties", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"use_path_style" = "$attributes.usePathStyle",
"provider" = "${getS3Provider()}"
)
diff --git a/regression-test/suites/load_p2/broker_load/test_s3_load_with_load_parallelism.groovy b/regression-test/suites/load_p2/broker_load/test_s3_load_with_load_parallelism.groovy
index c089f92d412..e81e7ab9b21 100644
--- a/regression-test/suites/load_p2/broker_load/test_s3_load_with_load_parallelism.groovy
+++ b/regression-test/suites/load_p2/broker_load/test_s3_load_with_load_parallelism.groovy
@@ -17,7 +17,8 @@
// under the License.
suite("test_s3_load_with_load_parallelism", "p2") {
-
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tableName = "test_load_parallelism"
def tableName1 = "paral_load"
@@ -95,8 +96,8 @@ suite("test_s3_load_with_load_parallelism", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
${prop}
diff --git a/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy b/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy
index cd27bb0906b..b8e7693b290 100644
--- a/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy
+++ b/regression-test/suites/load_p2/broker_load/tvf/test_tvf_based_broker_load.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_tvf_based_broker_load", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = ["part",
"upper_case",
"reverse",
@@ -50,38 +52,38 @@ suite("test_tvf_based_broker_load", "p2") {
"orc_s3_case9", // table column uppercase * load column lowercase * orc file uppercase
"csv_s3_case_line_delimiter" // csv format table with special line delimiter
]
- def paths = ["s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/path/*/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- "s3://doris-build-1308700295/regression/load/data/part*",
- // "s3://doris-build-1308700295/regression/load/data/random_all_types/part*", // just ignore it, parquet_case9 can't support complex type
- "s3://doris-build-1308700295/regression/load/data/orc/hits_100k_rows.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_lowercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/load/data/orc/hits_10k_rows_uppercase.orc",
- "s3://doris-build-1308700295/regression/line_delimiter/lineitem_0x7.csv.gz"
+ def paths = ["s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/path/*/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ "s3://${s3BucketName}/regression/load/data/part*",
+ // "s3://${s3BucketName}/regression/load/data/random_all_types/part*", // just ignore it, parquet_case9 can't support complex type
+ "s3://${s3BucketName}/regression/load/data/orc/hits_100k_rows.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_lowercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/load/data/orc/hits_10k_rows_uppercase.orc",
+ "s3://${s3BucketName}/regression/line_delimiter/lineitem_0x7.csv.gz"
]
def columns_list = ["""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
"""p_partkey, p_name, p_mfgr, p_brand, p_type, p_size, p_container, p_retailprice, p_comment""",
@@ -230,8 +232,8 @@ suite("test_tvf_based_broker_load", "p2") {
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
)
"""
@@ -241,7 +243,7 @@ suite("test_tvf_based_broker_load", "p2") {
}
def etl_info = ["unselected.rows=0; dpp.abnorm.ALL=0; dpp.norm.ALL=200000"]
- def task_info = ["cluster:cos.ap-beijing.myqcloud.com; timeout(s):14400; max_filter_ratio:0.0"]
+ def task_info = ["cluster:${s3Endpoint}; timeout(s):14400; max_filter_ratio:0.0"]
def error_msg = [""]
// test load
diff --git a/regression-test/suites/load_p2/tvf/test_s3_tvf.groovy b/regression-test/suites/load_p2/tvf/test_s3_tvf.groovy
index 4fd525d4314..8c09187a44b 100644
--- a/regression-test/suites/load_p2/tvf/test_s3_tvf.groovy
+++ b/regression-test/suites/load_p2/tvf/test_s3_tvf.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_s3_tvf", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
def tables = [
"agg_tbl_basic_tvf",
"dup_tbl_array_tvf",
@@ -59,7 +61,7 @@ suite("test_s3_tvf", "p2") {
// this parameter is similar with columns from path as (c1, c2, ...) in broker_load method, look at test case test_tvf_based_broker_load.groovy
for (String table : basicTables) {
attributeList.add(new TvfAttribute(table, "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, kd16", "K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18,Kd16", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/kd16=abcdefg/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("use_path_style", "true")
@@ -69,7 +71,7 @@ suite("test_s3_tvf", "p2") {
// SET (k19=to_bitmap(c6), k20=HLL_HASH(c6), k21=TO_QUANTILE_STATE(c5, 1.0), kd19=to_bitmap(c6), kd20=HLL_HASH(c6), kd21=TO_QUANTILE_STATE(c5, 1.0)
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21, kd16",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21,kd16", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/kd16=abcdefg/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("use_path_style", "true")
@@ -78,7 +80,7 @@ suite("test_s3_tvf", "p2") {
// path partition key maybe don't support array type ?
// for(String table : arrayTables) {
// attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "kd16"], "", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=[1,2,3,4]/basic_array_data.csv")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/kd16=[1,2,3,4]/basic_array_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("force_parsing_by_standard_uri", "true")
@@ -87,7 +89,7 @@ suite("test_s3_tvf", "p2") {
for (String table : basicTables) {
attributeList.add(new TvfAttribute(table, "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, kd16, kd17", "K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18,Kd16, kd17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/kd17=hello/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/kd16=abcdefg/kd17=hello/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("use_path_style", "true")
@@ -97,7 +99,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21, kd16",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21,kd16", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/kd17=hello/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/kd16=abcdefg/kd17=hello/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("use_path_style", "true")
@@ -106,7 +108,7 @@ suite("test_s3_tvf", "p2") {
// for (String table : basicTables) {
// attributeList.add(new TvfAttribute(table, "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, kd16, kd17", "K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18,Kd16, kd17", "", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/*/basic_data.csv")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/kd16=abcdefg/*/basic_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true")
@@ -116,7 +118,7 @@ suite("test_s3_tvf", "p2") {
// attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21, kd16",
// "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21,kd16", "", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/*/basic_data.csv")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/kd16=abcdefg/*/basic_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true")
@@ -129,7 +131,7 @@ suite("test_s3_tvf", "p2") {
// trim_double_quotes
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -138,7 +140,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -146,7 +148,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -156,7 +158,7 @@ suite("test_s3_tvf", "p2") {
// trim_double_quotes with skip_lines : 1
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -166,7 +168,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -175,7 +177,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -186,7 +188,7 @@ suite("test_s3_tvf", "p2") {
// stip_outer_array
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -197,7 +199,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -207,7 +209,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -219,7 +221,7 @@ suite("test_s3_tvf", "p2") {
// read_json_by_line
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -231,7 +233,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -242,7 +244,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_trim_double_quotes.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true")
@@ -261,8 +263,8 @@ suite("test_s3_tvf", "p2") {
// select * from s3(
// "s3.access_key" = "$a3k",
// "s3.secret_key" = "$s3k",
- // "s3.region" = "ap-beijing",
- // "uri" = "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json",
+ // "s3.region" = "${s3Region}",
+ // "uri" = "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json",
// "format" = "json",
// "read_json_by_line" = "false",
// "strip_outer_array" = "true",
@@ -277,7 +279,7 @@ suite("test_s3_tvf", "p2") {
// jsonpaths
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "*","k00,k01,k12", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -289,7 +291,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02, k03,k04, k05,k19, k20, k21, kd19, kd20, kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -299,7 +301,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04", "k00,k01,k02,k03,k04", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -310,7 +312,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "*","k00,k01,k12", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "true")
@@ -324,7 +326,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02, k03,k04, k05,k19, k20, k21, kd19, kd20, kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -335,7 +337,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04", "k00,k01,k02,k03,k04", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -347,7 +349,7 @@ suite("test_s3_tvf", "p2") {
// json_root
attributeList.add(new TvfAttribute("nest_tbl_basic_tvf", "id, city, code", "id, city, code", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/nest_json.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/nest_json.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("json_root", '$.item')
@@ -356,7 +358,7 @@ suite("test_s3_tvf", "p2") {
// num_as_string
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -367,7 +369,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -377,7 +379,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -388,7 +390,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -398,7 +400,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -408,7 +410,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -420,7 +422,7 @@ suite("test_s3_tvf", "p2") {
// fuzzy_parse
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -430,7 +432,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -439,7 +441,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -454,7 +456,7 @@ suite("test_s3_tvf", "p2") {
/* normal */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -462,14 +464,14 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -477,7 +479,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -485,14 +487,14 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -500,27 +502,27 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|"))
}
// for(String table : basicTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
-// .addProperty("uri", "https://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv")
+// .addProperty("uri", "https://${s3Endpoint}/${s3BucketName}/regression/load/data/basic_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
@@ -528,14 +530,14 @@ suite("test_s3_tvf", "p2") {
//
// attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
// "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
-// .addProperty("uri", "https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+// .addProperty("uri", "https://${s3Endpoint}/${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
//
// for(String table : arrayTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
-// .addProperty("uri", "https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+// .addProperty("uri", "https://${s3Endpoint}/${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("use_path_style", "true"))
@@ -544,7 +546,7 @@ suite("test_s3_tvf", "p2") {
/* error */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "", true)
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -552,14 +554,14 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "", true)
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "kd01", "kd02", "kd03", "kd04", "kd05", "kd06", "kd07", "kd08", "kd09", "kd10", "kd11", "kd12", "kd13", "kd14", "kd15", "kd16"], "", "", true)
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -568,7 +570,7 @@ suite("test_s3_tvf", "p2") {
/* skip lines */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10")
@@ -577,7 +579,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10")
@@ -585,7 +587,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_with_errors.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("skip_lines", "10")
@@ -595,21 +597,21 @@ suite("test_s3_tvf", "p2") {
/* compress type */
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.gz")
.addProperty("format", "csv")
.addProperty("column_separator", "|").addProperty("compress_type", "GZ"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.gz")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "GZ"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.gz")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.gz")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "GZ"))
@@ -617,21 +619,21 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.bz2")
.addProperty("format", "csv")
.addProperty("column_separator", "|").addProperty("compress_type", "BZ2"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.bz2")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "BZ2"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.bz2")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.bz2")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "BZ2"))
@@ -639,21 +641,21 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.lz4")
.addProperty("format", "csv")
.addProperty("column_separator", "|").addProperty("compress_type", "LZ4FRAME"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.lz4")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZ4FRAME"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.lz4")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.lz4")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZ4FRAME"))
@@ -662,14 +664,14 @@ suite("test_s3_tvf", "p2") {
// compress_type: lzo
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lzo")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.lzo")
.addProperty("format", "csv")
.addProperty("column_separator", "|").addProperty("compress_type", "LZO"))
}
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lzo")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv.lzo")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZO"))
@@ -677,7 +679,7 @@ suite("test_s3_tvf", "p2") {
// we can upload basic_array_data.csv.lzo to s3
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- .addProperty("uri", "https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.lzo")
+ .addProperty("uri", "https://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.lzo")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZO"))
@@ -686,7 +688,7 @@ suite("test_s3_tvf", "p2") {
// compress_type: deflate
// for(String table : basicTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_test.csv.deflate")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_test.csv.deflate")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("compress_type", "DEFLATE")
@@ -695,7 +697,7 @@ suite("test_s3_tvf", "p2") {
// attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
// "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_test.csv.deflate")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_test.csv.deflate")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("compress_type", "DEFLATE")
@@ -703,7 +705,7 @@ suite("test_s3_tvf", "p2") {
// for(String table : arrayTables) {
// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.deflate")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv.deflate")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("compress_type", "DEFLATE")
@@ -712,7 +714,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "WHERE c1 > 50", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -720,14 +722,14 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"WHERE c1 > 50", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17"], "WHERE c1 > 50", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -735,7 +737,7 @@ suite("test_s3_tvf", "p2") {
for(String table : uniqTable) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14", "k15", "k16", "k17", "k18"], "", "ORDER BY c1")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -743,7 +745,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -751,14 +753,14 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.parq")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.parq")
.addProperty("format", "parquet")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -766,7 +768,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -774,14 +776,14 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.orc")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.orc")
.addProperty("format", "orc")
.addProperty("column_separator", "|")
.addProperty("force_parsing_by_standard_uri", "true"))
@@ -789,7 +791,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -799,7 +801,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -808,7 +810,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "false")
.addProperty("strip_outer_array", "true")
@@ -818,7 +820,7 @@ suite("test_s3_tvf", "p2") {
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -828,7 +830,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18, to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -837,7 +839,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_by_line.json")
.addProperty("format", "json")
.addProperty("read_json_by_line", "true")
.addProperty("strip_outer_array", "false")
@@ -850,7 +852,7 @@ suite("test_s3_tvf", "p2") {
// line_delimiter: \t
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line_delimiter.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line_delimiter.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("line_delimiter", "\t")
@@ -859,7 +861,7 @@ suite("test_s3_tvf", "p2") {
attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line_delimiter.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data_by_line_delimiter.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("line_delimiter", "\t")
@@ -867,7 +869,7 @@ suite("test_s3_tvf", "p2") {
for(String table : arrayTables) {
attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "")
- .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_tab_line_delimiter.csv")
+ .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data_by_tab_line_delimiter.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
.addProperty("line_delimiter", "\t")
@@ -877,7 +879,7 @@ suite("test_s3_tvf", "p2") {
// invalid line delimiter, this will case error
// for(String table : basicTables) {
// attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17", "K18"], "", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("line_delimiter", ",")
@@ -886,7 +888,7 @@ suite("test_s3_tvf", "p2") {
// attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
// "k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21" ,"", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("line_delimiter", ",")
@@ -894,7 +896,7 @@ suite("test_s3_tvf", "p2") {
// for(String table : arrayTables) {
// attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14", "K15", "K16", "K17"], "", "")
- // .addProperty("uri", "s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ // .addProperty("uri", "s3://${s3BucketName}.${s3Endpoint}/regression/load/data/basic_array_data.csv")
// .addProperty("format", "csv")
// .addProperty("column_separator", "|")
// .addProperty("line_delimiter", ",")
@@ -923,7 +925,7 @@ suite("test_s3_tvf", "p2") {
FROM S3 (
"s3.access_key" = "$ak",
"s3.secret_key" = "$sk",
- "s3.region" = "ap-beijing",
+ "s3.region" = "${s3Region}",
${prop}
) ${attribute.whereClause}
${attribute.orderByClause}
diff --git a/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy b/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy
index 2167c40f1d2..df7ce235e85 100644
--- a/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy
+++ b/regression-test/suites/partition_p2/auto_partition/diff_data/stress_test_diff_date_list.groovy
@@ -28,7 +28,7 @@ suite("stress_test_diff_date_list", "p2,nonConcurrent") {
// get doris-db from s3
def dirPath = context.file.parent
def fileName = "doris-dbgen"
- def fileUrl = "http://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen"
+ def fileUrl = "http://${getS3BucketName()}.${getS3Endpoint()}/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen"
def filePath = Paths.get(dirPath, fileName)
if (!Files.exists(filePath)) {
new URL(fileUrl).withInputStream { inputStream ->
diff --git a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy
index fb400105758..c8bbdfbffc4 100644
--- a/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy
+++ b/regression-test/suites/partition_p2/auto_partition/same_data/stress_test_same_date_range.groovy
@@ -28,7 +28,7 @@ suite("stress_test_same_date_range", "p2,nonConcurrent") {
// get doris-db from s3
def dirPath = context.file.parent
def fileName = "doris-dbgen"
- def fileUrl = "http://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen"
+ def fileUrl = "http://${getS3BucketName()}.${getS3Endpoint()}/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen"
def filePath = Paths.get(dirPath, fileName)
if (!Files.exists(filePath)) {
new URL(fileUrl).withInputStream { inputStream ->
diff --git a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy
index 478ee0d7124..ecbb277c5fd 100644
--- a/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy
+++ b/regression-test/suites/partition_p2/auto_partition/two_stream_load/stress_test_two_stream_load.groovy
@@ -26,7 +26,7 @@ suite("stress_test_two_stream_load", "p2,nonConcurrent") {
// get doris-db from s3
def dirPath = context.file.parent
def fileName = "doris-dbgen"
- def fileUrl = "http://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen"
+ def fileUrl = "http://${getS3BucketName()}.${getS3Endpoint()}/regression/doris-dbgen-23-10-18/doris-dbgen-23-10-20/doris-dbgen"
def filePath = Paths.get(dirPath, fileName)
if (!Files.exists(filePath)) {
new URL(fileUrl).withInputStream { inputStream ->
diff --git a/regression-test/suites/query_profile/s3_load_profile_test.groovy b/regression-test/suites/query_profile/s3_load_profile_test.groovy
index 33bdf48895a..686e357d4e1 100644
--- a/regression-test/suites/query_profile/s3_load_profile_test.groovy
+++ b/regression-test/suites/query_profile/s3_load_profile_test.groovy
@@ -29,6 +29,8 @@ def getProfile = { id ->
// ref https://github.com/apache/doris/blob/3525a03815814f66ec78aa2ad6bbd9225b0e7a6b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
suite('s3_load_profile_test') {
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
sql "drop table if exists dup_tbl_basic;"
sql """
CREATE TABLE dup_tbl_basic
@@ -97,7 +99,7 @@ PROPERTIES (
"replication_num" = "1"
);
"""
- def loadAttribute =new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ def loadAttribute =new LoadAttributes("s3://${getS3BucketName()}/regression/load/data/basic_data.csv",
"dup_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
"", "", "", "", "")
@@ -128,8 +130,8 @@ PROPERTIES (
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"use_path_style" = "$loadAttribute.usePathStyle",
"provider" = "${getS3Provider()}"
)
diff --git a/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy b/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy
index a589a484d5e..ef927ddaf43 100644
--- a/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy
+++ b/regression-test/suites/statistics/test_update_rows_and_partition_first_load.groovy
@@ -16,7 +16,9 @@
// under the License.
suite("test_update_rows_and_partition_first_load", "p2") {
-
+ def s3BucketName = getS3BucketName()
+ def s3Endpoint = getS3Endpoint()
+ def s3Region = getS3Region()
String ak = getS3AK()
String sk = getS3SK()
String enabled = context.config.otherConfigs.get("enableBrokerLoad")
@@ -88,24 +90,24 @@ suite("test_update_rows_and_partition_first_load", "p2") {
def label = "part_" + UUID.randomUUID().toString().replace("-", "0")
sql """
LOAD LABEL ${label} (
- DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_1.csv")
+ DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_1.csv")
INTO TABLE update_rows_test1
COLUMNS TERMINATED BY ",",
- DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_2.csv")
+ DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_2.csv")
INTO TABLE update_rows_test2
COLUMNS TERMINATED BY ",",
- DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_1.csv")
+ DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_1.csv")
INTO TABLE partition_test1
COLUMNS TERMINATED BY ",",
- DATA INFILE("s3://doris-build-1308700295/regression/load/data/update_rows_2.csv")
+ DATA INFILE("s3://${s3BucketName}/regression/load/data/update_rows_2.csv")
INTO TABLE partition_test2
COLUMNS TERMINATED BY ","
)
WITH S3 (
"AWS_ACCESS_KEY" = "$ak",
"AWS_SECRET_KEY" = "$sk",
- "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
- "AWS_REGION" = "ap-beijing",
+ "AWS_ENDPOINT" = "${s3Endpoint}",
+ "AWS_REGION" = "${s3Region}",
"provider" = "${getS3Provider()}"
);
"""
diff --git a/regression-test/suites/tpcds_sf1000_p2/load.groovy b/regression-test/suites/tpcds_sf1000_p2/load.groovy
index aaf4fd54d71..9bf888e93b0 100644
--- a/regression-test/suites/tpcds_sf1000_p2/load.groovy
+++ b/regression-test/suites/tpcds_sf1000_p2/load.groovy
@@ -21,12 +21,13 @@
*
*/
suite("load") {
+ def s3Region = getS3Region()
restore {
location "s3://${getS3BucketName()}/regression/tpcds/sf1000"
ak "${getS3AK()}"
sk "${getS3SK()}"
endpoint "http://${getS3Endpoint()}"
- region "ap-beijing"
+ region "${s3Region}"
repository "tpcds_backup"
snapshot "tpcds_customer"
timestamp "2022-03-31-10-16-46"
@@ -40,7 +41,7 @@ suite("load") {
ak "${getS3AK()}"
sk "${getS3SK()}"
endpoint "http://${getS3Endpoint()}"
- region "ap-beijing"
+ region "${s3Region}"
repository "tpcds_backup"
snapshot "tpcds"
timestamp "2022-03-30-12-22-31"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
(doris) 02/02: [fix](nerieds) set row count to colStats if colStats.isUnknown (#37522)
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d643e85a7fb47e95f19408195fdb506ae1f2fb29
Author: minghong <en...@gmail.com>
AuthorDate: Thu Jul 11 10:48:46 2024 +0800
[fix](nerieds) set row count to colStats if colStats.isUnknown (#37522)
---
.../main/java/org/apache/doris/nereids/stats/StatsCalculator.java | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 8785be7eb93..8bc71e936b2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -482,6 +482,9 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
for (SlotReference slot : outputSlotReferences) {
ColumnStatistic cache = getColumnStatsFromTableCache(olapScan, slot);
ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(cache);
+ if (cache.isUnKnown) {
+ colStatsBuilder.setCount(rowCount);
+ }
adjustColStats(olapScan, slot, colStatsBuilder);
builder.putColumnStatistics(slot, colStatsBuilder.build());
rowCount = Math.max(rowCount, colStatsBuilder.getCount());
@@ -1051,6 +1054,9 @@ public class StatsCalculator extends DefaultPlanVisitor<Statistics, Void> {
for (SlotReference slot : slotSet) {
ColumnStatistic cache = getColumnStatsFromTableCache(catalogRelation, slot);
ColumnStatisticBuilder colStatsBuilder = new ColumnStatisticBuilder(cache);
+ if (cache.isUnKnown) {
+ colStatsBuilder.setCount(rowCount);
+ }
adjustColStats(catalogRelation, slot, colStatsBuilder);
rowCount = Math.max(rowCount, colStatsBuilder.getCount());
builder.putColumnStatistics(slot, colStatsBuilder.build());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org