You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by kx...@apache.org on 2023/10/24 14:02:12 UTC
[doris] 12/12: [regression](multi-catalog) add aliyun dlf hive on oss and huawei obs test case (#25650)
This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4213b19415accfb074c0fbd117deea2ac9e9ac70
Author: slothever <18...@users.noreply.github.com>
AuthorDate: Tue Oct 24 20:52:50 2023 +0800
[regression](multi-catalog) add aliyun dlf hive on oss and huawei obs test case (#25650)
add aliyun dlf hive on oss and huawei obs test case
now obs cases have some problem, will not fix this at this PR, just add comment.
---
regression-test/conf/regression-conf.groovy | 4 ++
.../hive/test_cloud_accessible_obs.out | 7 +++
.../hive/test_cloud_accessible_oss.out | 10 +++
.../hive/test_cloud_accessible_obs.groovy | 47 ++++++++++++++
.../hive/test_cloud_accessible_oss.groovy | 73 ++++++++++++++++++++++
5 files changed, 141 insertions(+)
diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy
index 9da502b8144..141dc57ed6b 100644
--- a/regression-test/conf/regression-conf.groovy
+++ b/regression-test/conf/regression-conf.groovy
@@ -151,9 +151,13 @@ extEsPort = 9200
extEsUser = "*******"
extEsPassword = "***********"
+enableObjStorageTest=false
enableMaxComputeTest=false
aliYunAk="***********"
+dlfUid="***********"
aliYunSk="***********"
+hwYunAk="***********"
+hwYunSk="***********"
s3Endpoint = "cos.ap-hongkong.myqcloud.com"
s3BucketName = "doris-build-hk-1308700295"
diff --git a/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out b/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out
new file mode 100644
index 00000000000..2a935c23e0f
--- /dev/null
+++ b/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out
@@ -0,0 +1,7 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !hms_q1 --
+123 22 234 123.324235 sawer 234.1234 a23f 1234vb false 2023-04-23 21:23:34.123 2023-04-23
+
+-- !hms_q2 --
+11625 62 234 2347.6236 awsali 546.2342 a23f 1234vb true 2023-04-22 21:21:34.123 2023-04-21 2023-04-24
+23621 23 234 345.12512356 aswwas 525.2352 a23f wsd false 2023-04-23 21:23:34.123 2023-04-23 2023-04-22
diff --git a/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out b/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out
new file mode 100644
index 00000000000..97933cfc5d4
--- /dev/null
+++ b/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out
@@ -0,0 +1,10 @@
+-- This file is automatically generated. You should know what you did if you want to edit this
+-- !hms_q1 --
+123 22 234 123.324235 sawer 234.1234 a23f 1234vb false 2023-04-23 21:23:34.123 2023-04-23
+
+-- !hms_q2 --
+11625 62 234 2347.6236 awsali 546.2342 a23f 1234vb true 2023-04-22 21:21:34.123 2023-04-21 2023-04-24
+23621 23 234 345.12512356 aswwas 525.2352 a23f wsd false 2023-04-23 21:23:34.123 2023-04-23 2023-04-22
+
+-- !dlf_q1 --
+26 AAAAAAAAKBAAAAAA 1997-08-16 2000-08-15 site_4 2450594 2446944 Unknown William Carter 2 Also difficult women relax. Raw solutions exclude Often well-known police can hand acres. Healthy, sensible problems make more silent recordings Keith Williams 2 able 892 Cedar North Drive Suite 280 Midway Williamson County TN 31904 United States -5.00 0.07
diff --git a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy
new file mode 100644
index 00000000000..b475088bd56
--- /dev/null
+++ b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cloud_accessible_obs", "p2,external,hive,external_remote,external_remote_hive") {
+ String enabled = context.config.otherConfigs.get("enableObjStorageTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort")
+ String ak = context.config.otherConfigs.get("hwYunAk")
+ String sk = context.config.otherConfigs.get("hwYunSk")
+ String hms_catalog_name = "test_cloud_accessible_obs"
+ sql """drop catalog if exists ${hms_catalog_name};"""
+ sql """
+ CREATE CATALOG IF NOT EXISTS ${hms_catalog_name}
+ PROPERTIES (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}',
+ 'obs.endpoint' = 'obs.cn-north-4.myhuaweicloud.com',
+ 'obs.access_key' = '${ak}',
+ 'obs.secret_key' = '${sk}'
+ );
+ """
+
+ logger.info("catalog " + hms_catalog_name + " created")
+ sql """switch ${hms_catalog_name};"""
+ logger.info("switched to catalog " + hms_catalog_name)
+ sql """ use cloud_accessible """
+ qt_hms_q1 """ select * from types_obs order by hms_int """
+ qt_hms_q2 """ select * from types_one_part_obs order by hms_int """
+
+ sql """drop catalog ${hms_catalog_name};"""
+ }
+}
diff --git a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy
new file mode 100644
index 00000000000..d7319a7ed08
--- /dev/null
+++ b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_cloud_accessible_oss", "p2,external,hive,external_remote,external_remote_hive") {
+ String enabled = context.config.otherConfigs.get("enableObjStorageTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost")
+ String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort")
+ String ak = context.config.otherConfigs.get("aliYunAk")
+ String sk = context.config.otherConfigs.get("aliYunSk")
+ String hms_catalog_name = "test_cloud_accessible_oss"
+ sql """drop catalog if exists ${hms_catalog_name};"""
+ sql """
+ CREATE CATALOG IF NOT EXISTS ${hms_catalog_name}
+ PROPERTIES (
+ 'type' = 'hms',
+ 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}',
+ 'oss.endpoint' = 'oss-cn-beijing.aliyuncs.com',
+ 'oss.access_key' = '${ak}',
+ 'oss.secret_key' = '${sk}'
+ );
+ """
+
+ logger.info("catalog " + hms_catalog_name + " created")
+ sql """switch ${hms_catalog_name};"""
+ logger.info("switched to catalog " + hms_catalog_name)
+ sql """ use cloud_accessible """
+ qt_hms_q1 """ select * from types_oss order by hms_int """
+ qt_hms_q2 """ select * from types_one_part_oss order by hms_int """
+
+ sql """drop catalog ${hms_catalog_name};"""
+
+ // dlf case
+ String dlf_catalog_name = "test_cloud_accessible_dlf"
+ String dlf_uid = context.config.otherConfigs.get("dlfUid")
+ sql """drop catalog if exists ${dlf_catalog_name};"""
+ sql """
+ CREATE CATALOG IF NOT EXISTS ${dlf_catalog_name}
+ PROPERTIES (
+ "type"="hms",
+ "hive.metastore.type" = "dlf",
+ "dlf.endpoint" = "dlf.cn-beijing.aliyuncs.com",
+ "dlf.region" = "cn-beijing",
+ "dlf.proxy.mode" = "DLF_ONLY",
+ "dlf.uid" = "${dlf_uid}",
+ "dlf.access_key" = "${ak}",
+ "dlf.secret_key" = "${sk}",
+ "dlf.access.public" = "true"
+ );
+ """
+ logger.info("catalog " + dlf_catalog_name + " created")
+ sql """switch ${dlf_catalog_name};"""
+ logger.info("switched to catalog " + dlf_catalog_name)
+ sql """ use jz_datalake """
+ qt_dlf_q1 """ select * from web_site where web_site_id='AAAAAAAAKBAAAAAA' order by web_site_sk,web_site_id limit 1; """ // test char,date,varchar,double,decimal
+
+ sql """drop catalog ${dlf_catalog_name};"""
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org