You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/23 11:05:52 UTC
(doris) 12/13: [regression](cold_heat)add cold heat test for hdfs (#31074)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 30ef9b70c3a5768275c169bbdce91df27b571f4b
Author: lsy3993 <11...@users.noreply.github.com>
AuthorDate: Fri Feb 23 15:45:58 2024 +0800
[regression](cold_heat)add cold heat test for hdfs (#31074)
* add cold heat test for hdfs
---
.../alter_table_add_policy_by_hdfs.groovy | 145 ++++++
...reate_table_use_partition_policy_by_hdfs.groovy | 166 +++++++
.../create_table_use_policy_by_hdfs.groovy | 108 +++++
.../disable_storage_policy_MoW_by_hdfs.groovy | 137 ++++++
.../modify_partition_add_policy_by_hdfs.groovy | 158 +++++++
.../use_default_storage_policy_bu_hdfs.groovy | 105 +++++
.../policy/alter_hdfs_resouce.groovy | 262 +++++++++++
.../policy/create_hdfs_resource.groovy | 456 +++++++++++++++++++
.../policy/drop_hdfs_reource.groovy | 148 ++++++
.../policy/show_hdfs_resource.groovy | 70 +++
.../add_drop_partition_by_hdfs.groovy | 284 ++++++++++++
...eate_table_use_dynamic_partition_by_hdfs.groovy | 279 ++++++++++++
...reate_table_use_partition_policy_by_hdfs.groovy | 370 +++++++++++++++
.../create_table_use_policy_by_hdfs.groovy | 294 ++++++++++++
.../modify_replica_use_partition_by_hdfs.groovy | 506 +++++++++++++++++++++
.../table_modify_resouce_and_policy_by_hdfs.groovy | 326 +++++++++++++
16 files changed, 3814 insertions(+)
diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/alter_table_add_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/alter_table_add_policy_by_hdfs.groovy
new file mode 100644
index 00000000000..2517c2904b9
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/alter_table_add_policy_by_hdfs.groovy
@@ -0,0 +1,145 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("add_table_policy_by_alter_table_hdfs") {
+ def create_table_not_have_policy_result = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_not_have_policy_hdfs
+ (
+ k1 BIGINT,
+ k2 LARGEINT,
+ v1 VARCHAR(2048)
+ )
+ UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH (k1) BUCKETS 3
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+ assertEquals(create_table_not_have_policy_result.size(), 1);
+
+ def alter_table_use_not_exist_policy_result = try_sql """
+ ALTER TABLE create_table_not_have_policy_hdfs set ("storage_policy" = "policy_not_exist");
+ """
+ // errCode = 2, detailMessage = Resource does not exist. name: policy_not_exist
+ assertEquals(alter_table_use_not_exist_policy_result, null);
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ if (!storage_exist.call("created_create_table_alter_policy_hdfs")) {
+ def create_hdfs_resource = try_sql """
+ CREATE RESOURCE IF NOT EXISTS "test_create_alter_table_use_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS created_create_table_alter_policy_hdfs
+ PROPERTIES(
+ "storage_resource" = "test_create_alter_table_use_resource_hdfs",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ assertEquals(storage_exist.call("created_create_table_alter_policy_hdfs"), true)
+ }
+
+ def alter_table_try_again_result = try_sql """
+ ALTER TABLE create_table_not_have_policy_hdfs set ("storage_policy" = "created_create_table_alter_policy_hdfs");
+ """
+ // OK
+ assertEquals(alter_table_try_again_result.size(), 1);
+
+ def alter_table_when_table_has_storage_policy_result = try_sql """
+ ALTER TABLE create_table_not_have_policy_hdfs set ("storage_policy" = "created_create_table_alter_policy_hdfs");
+ """
+ // OK
+ assertEquals(alter_table_when_table_has_storage_policy_result.size(), 1);
+
+ if (!storage_exist.call("created_create_table_alter_policy_1_hdfs")) {
+ def create_hdfs_resource = try_sql """
+ CREATE RESOURCE IF NOT EXISTS "test_create_alter_table_use_resource_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS created_create_table_alter_policy_1_hdfs
+ PROPERTIES(
+ "storage_resource" = "test_create_alter_table_use_resource_1_hdfs",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ assertEquals(storage_exist.call("created_create_table_alter_policy_1_hdfs"), true)
+ }
+
+ // current not support change storage policy name
+ def cannot_modify_exist_storage_policy_table_result = try_sql """
+ ALTER TABLE create_table_not_have_policy_hdfs set ("storage_policy" = "created_create_table_alter_policy_1_hdfs");
+ """
+ // OK
+ assertEquals(cannot_modify_exist_storage_policy_table_result.size(), 1);
+
+ // you can change created_create_table_alter_policy's policy cooldown time, cooldown ttl property,
+ // by alter storage policy
+ def modify_storage_policy_property_result = try_sql """
+ ALTER STORAGE POLICY "created_create_table_alter_policy_1_hdfs" PROPERTIES("cooldown_datetime" = "2026-06-08 00:00:00");
+ """
+ // change s3 resource, ak、sk by alter resource
+ def modify_storage_policy_property_result_1 = try_sql """
+ ALTER RESOURCE "test_create_alter_table_use_resource_1_hdfs" PROPERTIES("dfs.nameservices" = "has_been_changed");
+ """
+
+ sql """
+ DROP TABLE create_table_not_have_policy_hdfs;
+ """
+
+ sql """
+ DROP STORAGE POLICY created_create_table_alter_policy_1_hdfs;
+ """
+
+ sql """
+ DROP RESOURCE test_create_alter_table_use_resource_1_hdfs;
+ """
+
+}
+
diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_partition_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_partition_policy_by_hdfs.groovy
new file mode 100644
index 00000000000..4e55834adea
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_partition_policy_by_hdfs.groovy
@@ -0,0 +1,166 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("create_table_use_partition_policy_hdfs") {
+ def cooldown_ttl = "10"
+
+ def create_table_partition_use_not_create_policy = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_partition_use_not_create_policy_hdfs
+ (
+ k1 DATE,
+ k2 INT,
+ V1 VARCHAR(2048) REPLACE
+ ) PARTITION BY RANGE (k1) (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01") ("storage_policy" = "not_exist_policy_1" ,"replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01") ("storage_policy" = "not_exist_policy_2" ,"replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1;
+ """
+
+ // errCode = 2, detailMessage = Storage policy does not exist. name: not_exist_policy_1
+ assertEquals(create_table_partition_use_not_create_policy, null)
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ if (!storage_exist.call("test_create_table_partition_use_policy_1_hdfs")) {
+ def create_hdfs_resource = try_sql """
+ CREATE RESOURCE IF NOT EXISTS "test_create_table_partition_use_resource_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS test_create_table_partition_use_policy_1_hdfs
+ PROPERTIES(
+ "storage_resource" = "test_create_table_partition_use_resource_1_hdfs",
+ "cooldown_ttl" = "$cooldown_ttl"
+ );
+ """
+ assertEquals(storage_exist.call("test_create_table_partition_use_policy_1_hdfs"), true)
+ }
+
+
+ if (!storage_exist.call("test_create_table_partition_use_policy_2_hdfs")) {
+ def create_hdfs_resource = try_sql """
+ CREATE RESOURCE IF NOT EXISTS "test_create_table_partition_use_resource_2_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS test_create_table_partition_use_policy_2_hdfs
+ PROPERTIES(
+ "storage_resource" = "test_create_table_partition_use_resource_2_hdfs",
+ "cooldown_ttl" = "$cooldown_ttl"
+ );
+ """
+ assertEquals(storage_exist.call("test_create_table_partition_use_policy_2_hdfs"), true)
+ }
+
+ // success
+ def create_table_partition_use_created_policy = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_hdfs
+ (
+ k1 DATE,
+ k2 INT,
+ V1 VARCHAR(2048) REPLACE
+ ) PARTITION BY RANGE (k1) (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01") ("storage_policy" = "test_create_table_partition_use_policy_1_hdfs" ,"replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01") ("storage_policy" = "test_create_table_partition_use_policy_2_hdfs" ,"replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1;
+ """
+
+ assertEquals(create_table_partition_use_created_policy.size(), 1);
+
+ sql """
+ DROP TABLE create_table_partition_use_created_policy_hdfs
+ """
+
+ def create_table_partition_use_created_policy_1 = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_1_hdfs
+ (
+ k1 DATEV2,
+ k2 INT,
+ V1 VARCHAR(2048) REPLACE
+ ) PARTITION BY RANGE (k1) (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01") ("storage_policy" = "test_create_table_partition_use_policy_1_hdfs" ,"replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01") ("storage_policy" = "test_create_table_partition_use_policy_2_hdfs" ,"replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1;
+ """
+
+ assertEquals(create_table_partition_use_created_policy_1.size(), 1);
+
+ sql """
+ DROP TABLE create_table_partition_use_created_policy_1_hdfs
+ """
+
+ def create_table_partition_use_created_policy_2 = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_partition_use_created_policy_2_hdfs
+ (
+ k1 DATETIMEV2(3),
+ k2 INT,
+ V1 VARCHAR(2048) REPLACE
+ ) PARTITION BY RANGE (k1) (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01 00:00:00.111") ("storage_policy" = "test_create_table_partition_use_policy_1_hdfs" ,"replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01 00:00:00.111") ("storage_policy" = "test_create_table_partition_use_policy_2_hdfs" ,"replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1;
+ """
+
+ assertEquals(create_table_partition_use_created_policy_2.size(), 1);
+
+ sql """
+ DROP TABLE create_table_partition_use_created_policy_2_hdfs;
+ """
+ sql """
+ DROP STORAGE POLICY test_create_table_partition_use_policy_1_hdfs;
+ """
+ sql """
+ DROP STORAGE POLICY test_create_table_partition_use_policy_2_hdfs;
+ """
+ sql """
+ DROP RESOURCE test_create_table_partition_use_resource_1_hdfs
+ """
+ sql """
+ DROP RESOURCE test_create_table_partition_use_resource_2_hdfs
+ """
+}
+
diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_policy_by_hdfs.groovy
new file mode 100644
index 00000000000..a81dc07256b
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/create_table_use_policy_by_hdfs.groovy
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("create_table_use_policy_hdfs") {
+ def cooldown_ttl = "10"
+
+ def create_table_use_not_create_policy = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_use_not_create_policy_hdfs
+ (
+ k1 BIGINT,
+ k2 LARGEINT,
+ v1 VARCHAR(2048)
+ )
+ UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH (k1) BUCKETS 3
+ PROPERTIES(
+ "storage_policy" = "not_exist_policy",
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+
+ // errCode = 2, detailMessage = Storage policy does not exist. name: not_exist_policy
+ assertEquals(create_table_use_not_create_policy, null)
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def create_hdfs_resource = try_sql """
+ CREATE RESOURCE IF NOT EXISTS "test_create_table_use_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS test_create_table_use_policy_hdfs
+ PROPERTIES(
+ "storage_resource" = "test_create_table_use_resource_hdfs",
+ "cooldown_ttl" = "$cooldown_ttl"
+ );
+ """
+
+ sql """ALTER STORAGE POLICY test_create_table_use_policy_hdfs PROPERTIES("cooldown_ttl" = "$cooldown_ttl")"""
+
+ assertEquals(storage_exist.call("test_create_table_use_policy_hdfs"), true)
+
+ // success
+ def create_table_use_created_policy = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_use_created_policy_hdfs
+ (
+ k1 BIGINT,
+ k2 LARGEINT,
+ v1 VARCHAR(2048)
+ )
+ UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH (k1) BUCKETS 3
+ PROPERTIES(
+ "storage_policy" = "test_create_table_use_policy_hdfs",
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+
+ assertEquals(create_table_use_created_policy.size(), 1);
+
+ sql """
+ DROP TABLE IF EXISTS create_table_use_created_policy_hdfs
+ """
+ sql """
+ DROP STORAGE POLICY test_create_table_use_policy_hdfs
+ """
+ sql """
+ DROP RESOURCE test_create_table_use_resource_hdfs
+ """
+
+}
+
diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/disable_storage_policy_MoW_by_hdfs.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/disable_storage_policy_MoW_by_hdfs.groovy
new file mode 100644
index 00000000000..d5131d9c2a7
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/disable_storage_policy_MoW_by_hdfs.groovy
@@ -0,0 +1,137 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("disable_storage_policy_MoW_hdfs"){
+ def s3_source_name = "default_s3_source_test"
+ def hdfs_source_name = "default_hdfs_source_test"
+ def storage_policy_name = "default_storage_policy_test_hdfs"
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ if(!storage_exist.call("${storage_policy_name}")){
+ def create_hdfs_resource = sql """
+ CREATE RESOURCE IF NOT EXISTS "${hdfs_source_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_storage_policy = sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${storage_policy_name} PROPERTIES(
+ "storage_resource" = "${hdfs_source_name}",
+ "cooldown_ttl" = "1008611"
+ );
+ """
+
+ create_hdfs_resource
+ create_storage_policy
+ }
+
+ def table_name_test_1 = "disable_storage_policy_on_mow1_hdfs"
+ def table_name_test_2 = "disable_storage_policy_on_mow2_hdfs"
+ def table_name_test_3 = "disable_storage_policy_on_mow3_hdfs"
+ //Test case I. Should panic when creates MoW table with storage policy
+ test{
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name_test_1} (
+ k1 DATE,
+ k2 INT,
+ V1 VARCHAR(2048)
+ ) ENGINE = OLAP
+ UNIQUE KEY(k1,k2)
+ PARTITION BY RANGE (k1)
+ (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01") ("storage_policy" = "${storage_policy_name}", "replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01") ("storage_policy" = "${storage_policy_name}", "replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+ )
+ """
+ exception "with storage policy(${storage_policy_name})"
+ }
+
+ //Test case II. Should panic when sets storage policy to MoW table or its partitions
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name_test_2} (
+ id INT,
+ v1 INT
+ ) ENGINE = OLAP
+ UNIQUE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+ );
+ """
+
+ test {
+ sql """
+ ALTER TABLE ${table_name_test_2} MODIFY PARTITION(*) SET("storage_policy" = "${storage_policy_name}");
+ """
+ exception "with storage policy(${storage_policy_name})"
+ }
+
+ test {
+ sql """
+ ALTER TABLE ${table_name_test_2} SET ("storage_policy" = "${storage_policy_name}");
+ """
+ exception "with storage policy(${storage_policy_name})"
+ }
+
+ //Test case III. Should panic when creates MoW table(without partitions) with storage policy
+ test {
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name_test_3} (
+ k1 INT,
+ V1 INT
+ ) ENGINE = OLAP
+ UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH (k1) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "storage_policy" = "${storage_policy_name}"
+ );
+ """
+ exception "with storage policy(${storage_policy_name})"
+ }
+
+ //clean resource
+ sql""" DROP TABLE IF EXISTS ${table_name_test_1} """
+ sql""" DROP TABLE IF EXISTS ${table_name_test_2} """
+ sql""" DROP TABLE IF EXISTS ${table_name_test_3} """
+}
+
diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/modify_partition_add_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/modify_partition_add_policy_by_hdfs.groovy
new file mode 100644
index 00000000000..e6cbcd780ea
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/modify_partition_add_policy_by_hdfs.groovy
@@ -0,0 +1,158 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+suite("add_table_policy_by_modify_partition_hdfs") {
+ SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
+ Date date = new Date(System.currentTimeMillis() + 3600000)
+ def cooldownTime = format.format(date)
+
+ sql """DROP TABLE IF EXISTS create_table_partition_hdfs"""
+ def create_table_partition_not_have_policy_result = try_sql """
+ CREATE TABLE IF NOT EXISTS `create_table_partition_hdfs` (
+ `lo_orderkey` bigint(20) NOT NULL COMMENT "",
+ `lo_linenumber` bigint(20) NOT NULL COMMENT "",
+ `lo_custkey` int(11) NOT NULL COMMENT "",
+ `lo_partkey` int(11) NOT NULL COMMENT "",
+ `lo_suppkey` int(11) NOT NULL COMMENT "",
+ `lo_orderdate` int(11) NOT NULL COMMENT "",
+ `lo_orderpriority` varchar(16) NOT NULL COMMENT "",
+ `lo_shippriority` int(11) NOT NULL COMMENT "",
+ `lo_quantity` bigint(20) NOT NULL COMMENT "",
+ `lo_extendedprice` bigint(20) NOT NULL COMMENT "",
+ `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "",
+ `lo_discount` bigint(20) NOT NULL COMMENT "",
+ `lo_revenue` bigint(20) NOT NULL COMMENT "",
+ `lo_supplycost` bigint(20) NOT NULL COMMENT "",
+ `lo_tax` bigint(20) NOT NULL COMMENT "",
+ `lo_commitdate` bigint(20) NOT NULL COMMENT "",
+ `lo_shipmode` varchar(11) NOT NULL COMMENT ""
+ )
+ PARTITION BY RANGE(`lo_orderdate`)
+ (PARTITION p1992 VALUES [("-2147483648"), ("19930101")),
+ PARTITION p1998 VALUES [("19980101"), ("19990101")) )
+ DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 4
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+ assertEquals(create_table_partition_not_have_policy_result.size(), 1);
+
+ // support
+ // 1. ALTER TABLE create_table_partition MODIFY PARTITION p1992 SET("storage_policy"="not_exist_policy");
+ // 2. ALTER TABLE create_table_partition MODIFY PARTITION (p1992, p1998) SET("storage_policy"="not_exist_policy");
+ // 3. ALTER TABLE create_table_partition MODIFY PARTITION (*) SET("storage_policy"="not_exist_policy");
+ def alter_table_partition_use_not_exist_policy_result = try_sql """
+ ALTER TABLE create_table_partition_hdfs MODIFY PARTITION p1992 SET("storage_policy"="not_exist_policy");
+ """
+ // errCode = 2, detailMessage = Resource does not exist. name: not_exist_policy
+ assertEquals(alter_table_partition_use_not_exist_policy_result, null);
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def create_hdfs_resource = try_sql """
+ CREATE RESOURCE IF NOT EXISTS "test_modify_partition_table_use_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS created_create_table_partition_alter_policy_hdfs
+ PROPERTIES(
+ "storage_resource" = "test_modify_partition_table_use_resource_hdfs",
+ "cooldown_datetime" = "$cooldownTime"
+ );
+ """
+ sql """ALTER STORAGE POLICY created_create_table_partition_alter_policy_hdfs PROPERTIES("cooldown_datetime" = "$cooldownTime")"""
+ assertEquals(storage_exist.call("created_create_table_partition_alter_policy_hdfs"), true)
+
+ def alter_table_partition_try_again_result = try_sql """
+ ALTER TABLE create_table_partition_hdfs MODIFY PARTITION (*) SET("storage_policy"="created_create_table_partition_alter_policy_hdfs");
+ """
+ // OK
+ assertEquals(alter_table_partition_try_again_result.size(), 1);
+
+ try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS tmp2_hdfs
+ PROPERTIES(
+ "storage_resource" = "test_modify_partition_table_use_resource_hdfs",
+ "cooldown_datetime" = "$cooldownTime"
+ );
+ """
+
+ sql """
+ CREATE TABLE create_table_partion_use_created_policy_test_hdfs
+ (
+ k1 DATE,
+ k2 INT,
+ V1 VARCHAR(2048) REPLACE
+ ) PARTITION BY RANGE (k1) (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01") ("storage_policy" = "tmp2_hdfs" ,"replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01") ("storage_policy" = "tmp2_hdfs" ,"replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_policy" = "created_create_table_partition_alter_policy_hdfs"
+ );
+ """
+
+ // Test that the partition's specified policy would be covered by the table's policy
+ def partitions = sql """
+ show partitions from create_table_partion_use_created_policy_test_hdfs
+ """
+
+ for (par in partitions) {
+ assertTrue(par[12] == "created_create_table_partition_alter_policy_hdfs")
+ }
+
+ sql """
+ DROP TABLE IF EXISTS create_table_partition_hdfs;
+ """
+ sql """
+ DROP TABLE IF EXISTS create_table_partion_use_created_policy_test_hdfs;
+ """
+ sql """
+ DROP STORAGE POLICY created_create_table_partition_alter_policy_hdfs
+ """
+ sql """
+ DROP STORAGE POLICY tmp2_hdfs
+ """
+ sql """
+ DROP RESOURCE test_modify_partition_table_use_resource_hdfs
+ """
+}
+
diff --git a/regression-test/suites/cold_heat_separation/empty_table_use_policy/use_default_storage_policy_bu_hdfs.groovy b/regression-test/suites/cold_heat_separation/empty_table_use_policy/use_default_storage_policy_bu_hdfs.groovy
new file mode 100644
index 00000000000..48417febae4
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/empty_table_use_policy/use_default_storage_policy_bu_hdfs.groovy
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("use_default_storage_policy_hdfs") {
+ try_sql """
+ DROP TABLE use_default_storage_policy_hdfs;
+ """
+ try_sql """
+ DROP STORAGE POLICY default_storage_policy_hdfs
+ """
+ try_sql """
+ DROP RESOURCE default_hdfs_resource_hdfs
+ """
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ if (!storage_exist.call("default_storage_policy_hdfs")) {
+ def create_table_use_default_policy_but_not_set_default_policy_result = try_sql """
+ CREATE TABLE IF NOT EXISTS use_default_storage_policy_hdfs
+ ( k1 DATE, k2 INT, V1 VARCHAR(2048) REPLACE )
+ PARTITION BY RANGE (k1)
+ (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01") ("storage_policy" = "default_storage_policy_hdfs", "replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01") ("storage_policy" = "default_storage_policy_hdfs", "replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1;
+ """
+ // errCode = 2, detailMessage = Use default storage policy, but not give s3 info, please use alter resource to add default storage policy S3 info.
+ assertEquals(create_table_use_default_policy_but_not_set_default_policy_result, null);
+
+ def create_hdfs_resource = try_sql """
+ CREATE RESOURCE IF NOT EXISTS "default_hdfs_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS default_storage_policy_hdfs PROPERTIES(
+ "storage_resource" = "default_hdfs_resource_hdfs",
+ "cooldown_ttl" = "1008611"
+ );
+ """
+ assertEquals(storage_exist.call("default_storage_policy_hdfs"), true)
+ }
+
+ def create_table_use_default_policy_has_set_default_policy_result = try_sql """
+ CREATE TABLE IF NOT EXISTS use_default_storage_policy_hdfs
+ ( k1 DATE, k2 INT, V1 VARCHAR(2048) REPLACE )
+ PARTITION BY RANGE (k1)
+ (
+ PARTITION p1 VALUES LESS THAN ("2022-01-01") ("storage_policy" = "default_storage_policy_hdfs", "replication_num"="1"),
+ PARTITION p2 VALUES LESS THAN ("2022-02-01") ("storage_policy" = "default_storage_policy_hdfs", "replication_num"="1")
+ ) DISTRIBUTED BY HASH(k2) BUCKETS 1;
+ """
+ // success
+ assertEquals(create_table_use_default_policy_has_set_default_policy_result.size(), 1)
+
+ // you can change default_storage_policy's policy property, such as ak、sk,
+ // so table create_table_not_have_policy will use AWS_ACCESS_KEY = "has_been_changed"
+ def modify_storage_policy_property_result_1 = try_sql """
+ ALTER RESOURCE "default_hdfs_resource_hdfs" PROPERTIES("AWS_ACCESS_KEY" = "has_been_changed");
+ """
+
+ sql """
+ DROP TABLE use_default_storage_policy_hdfs;
+ """
+ sql """
+ DROP STORAGE POLICY default_storage_policy_hdfs
+ """
+ sql """
+ DROP RESOURCE default_hdfs_resource_hdfs
+ """
+}
+
diff --git a/regression-test/suites/cold_heat_separation/policy/alter_hdfs_resouce.groovy b/regression-test/suites/cold_heat_separation/policy/alter_hdfs_resouce.groovy
new file mode 100644
index 00000000000..44b38a41843
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/policy/alter_hdfs_resouce.groovy
@@ -0,0 +1,262 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("alter_hdfs_policy") {
+
+ def check_resource_delete_if_exist = { resource_name ->
+ def has_resouce = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ if (has_resouce.size() > 0) {
+ sql """
+ DROP RESOURCE '${resource_name}';
+ """
+ }
+ }
+
+ // create resource
+ def create_source = { resource_name ->
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ }
+
+ def alter_resource_change_property = { resource_name ->
+ // support
+ def alter_result_succ_1 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("fs.defaultFS" = "1111");
+ """
+
+ def alter_result_succ_2 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("hadoop.username" = "2222");
+ """
+
+ def alter_result_succ_5 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("hadoop.password" = "5555");
+ """
+
+ def alter_result_succ_6 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("dfs.nameservices" = "6666");
+ """
+
+ def alter_result_succ_7 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("dfs.ha.namenodes.my_ha" = "7777");
+ """
+
+ // errCode = 2, detailMessage = current not support modify property : dfs.namenode.rpc-address.my_ha.my_namenode1
+ def alter_result_fail_1 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("dfs.namenode.rpc-address.my_ha.my_namenode1" = "8888");
+ """
+
+ // errCode = 2, detailMessage = current not support modify property : dfs.namenode.rpc-address.my_ha.my_namenode2
+ def alter_result_fail_2 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("dfs.namenode.rpc-address.my_ha.my_namenode2" = "9999");
+ """
+
+ // errCode = 2, detailMessage = current not support modify property : dfs.client.failover.proxy.provider
+ def alter_result_fail_3 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("dfs.client.failover.proxy.provider" = "10101010");
+ """
+
+ // errCode = 2, detailMessage = current not support modify property : type
+ def alter_result_fail_5 = try_sql """
+ ALTER RESOURCE "${resource_name}" PROPERTIES("type" = "local");
+ """
+ }
+
+ // todo : this function should check by really situation
+ def check_alter_resource_result_no_policy = { resource_name ->
+ def show_alter_result = order_sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ logger.info(show_alter_result.toString())
+ println show_alter_result.toString()
+
+ // AWS_ACCESS_KEY
+ assertEquals(show_alter_result[1][3], "false")
+ // AWS_BUCKET
+ assertEquals(show_alter_result[2][3], "7777")
+ // AWS_MAX_CONNECTIONS
+ assertEquals(show_alter_result[3][3], "8888")
+ // AWS_REQUEST_TIMEOUT_MS
+ assertEquals(show_alter_result[4][3], "9999")
+ // AWS_CONNECTION_TIMEOUT_MS
+ assertEquals(show_alter_result[5][3], "6666")
+ // AWS_ENDPOINT
+ assertEquals(show_alter_result[6][3], "1111")
+ // AWS_REGION
+ assertEquals(show_alter_result[7][3], "5555")
+ // s3_rootpath
+ assertEquals(show_alter_result[8][3], "2222")
+ // AWS_SECRET_KEY
+ assertEquals(show_alter_result[9][3], "hdfs")
+ }
+
+ // todo : this function should check by really situation
+ def check_alter_resource_result_with_policy = { resource_name ->
+ def show_alter_result = order_sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ logger.info(show_alter_result.toString())
+ println show_alter_result.toString()
+
+ // AWS_ACCESS_KEY
+ assertEquals(show_alter_result[1][3], "false")
+ // AWS_BUCKET
+ assertEquals(show_alter_result[2][3], "7777")
+ // AWS_MAX_CONNECTIONS
+ assertEquals(show_alter_result[3][3], "8888")
+ // AWS_REQUEST_TIMEOUT_MS
+ assertEquals(show_alter_result[4][3], "9999")
+ // AWS_CONNECTION_TIMEOUT_MS
+ assertEquals(show_alter_result[5][3], "6666")
+ // AWS_ENDPOINT
+ assertEquals(show_alter_result[6][3], "1111")
+ // AWS_REGION
+ assertEquals(show_alter_result[7][3], "5555")
+ // s3_rootpath
+ assertEquals(show_alter_result[8][3], "2222")
+ // AWS_SECRET_KEY
+ assertEquals(show_alter_result[9][3], "hdfs")
+ }
+
+
+ // test when no policy binding to resource
+ def no_binding_policy_resource = "no_binding_policy_resource_hdfs"
+ check_resource_delete_if_exist(no_binding_policy_resource)
+ create_source(no_binding_policy_resource)
+ alter_resource_change_property(no_binding_policy_resource)
+ check_alter_resource_result_no_policy(no_binding_policy_resource)
+
+
+ // test when policy binding to resource
+ def has_resource_policy_alter = "has_resource_policy_alter_hdfs"
+ sql """
+ DROP STORAGE POLICY IF EXISTS has_resouce_policy_alter_policy_hdfs
+ """
+ sql """
+ DROP STORAGE POLICY IF EXISTS has_test_policy_to_alter_hdfs
+ """
+ sql """
+ DROP STORAGE POLICY IF EXISTS has_test_policy_to_alter_1_hdfs
+ """
+ check_resource_delete_if_exist(has_resource_policy_alter)
+ create_source(has_resource_policy_alter)
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS has_resouce_policy_alter_policy_hdfs
+ PROPERTIES(
+ "storage_resource" = "${has_resource_policy_alter}",
+ "cooldown_ttl" = "1d"
+ )
+ """
+ alter_resource_change_property(has_resource_policy_alter)
+ check_alter_resource_result_with_policy(has_resource_policy_alter)
+ sql """
+ DROP STORAGE POLICY has_resouce_policy_alter_policy_hdfs
+ """
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+
+ if (!storage_exist.call("has_test_policy_to_alter_hdfs")) {
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY has_test_policy_to_alter_hdfs
+ PROPERTIES(
+ "storage_resource" = "${has_resource_policy_alter}",
+ "cooldown_ttl" = "10086"
+ );
+ """
+ assertEquals(storage_exist.call("has_test_policy_to_alter_hdfs"), true)
+ }
+
+ // OK
+ def alter_result_sql_succ_ttl = try_sql """
+ ALTER STORAGE POLICY has_test_policy_to_alter_hdfs PROPERTIES("cooldown_ttl" = "10000");
+ """
+
+ // FAIL
+ alter_result_sql_succ_ttl = try_sql """
+ ALTER STORAGE POLICY has_test_policy_to_alter_hdfs PROPERTIES("cooldown_ttl" = "-10000");
+ """
+
+ // OK
+ def alter_result_sql_succ_datetime = try_sql """
+ ALTER STORAGE POLICY has_test_policy_to_alter_hdfs PROPERTIES("cooldown_datetime" = "2023-06-08 00:00:00");
+ """
+
+ // FAILED
+ alter_result_sql_succ_datetime = try_sql """
+ ALTER STORAGE POLICY has_test_policy_to_alter_hdfs PROPERTIES("cooldown_datetime" = "2023-13-32 00:00:00");
+ """
+
+ // FAILED
+ def alter_result_sql_failed_storage_resource = try_sql """
+ ALTER STORAGE POLICY has_test_policy_to_alter_hdfs PROPERTIES("storage_resource" = "${no_binding_policy_resource}");
+ """
+ // errCode = 2, detailMessage = not support change storage policy's storage resource, you can change s3 properties by alter resource
+ assertEquals(alter_result_sql_failed_storage_resource, null)
+
+ if (!storage_exist.call("has_test_policy_to_alter_1_hdfs")) {
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY has_test_policy_to_alter_1_hdfs
+ PROPERTIES(
+ "storage_resource" = "${no_binding_policy_resource}",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ assertEquals(storage_exist.call("has_test_policy_to_alter_1_hdfs"), true)
+ }
+
+ // go to check be、fe log about notify alter.
+ def alter_result_succ_again = try_sql """
+ ALTER RESOURCE "${has_resource_policy_alter}" PROPERTIES("AWS_ACCESS_KEY" = "akakak");
+ """
+
+ sql """
+ DROP STORAGE POLICY has_test_policy_to_alter_hdfs;
+ """
+
+ sql """
+ DROP STORAGE POLICY has_test_policy_to_alter_1_hdfs;
+ """
+
+ sql """
+ DROP RESOURCE '${no_binding_policy_resource}'
+ """
+}
+
diff --git a/regression-test/suites/cold_heat_separation/policy/create_hdfs_resource.groovy b/regression-test/suites/cold_heat_separation/policy/create_hdfs_resource.groovy
new file mode 100644
index 00000000000..0c903f3467e
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/policy/create_hdfs_resource.groovy
@@ -0,0 +1,456 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("create_hdfs_policy") {
+ def has_created_1 = sql """
+ SHOW RESOURCES WHERE NAME = "crete_policy_1_hdfs";
+ """
+
+ // doesn't have name crete_policy_1 resources
+ // normal
+ if(has_created_1.size() == 0) {
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "crete_policy_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ def create_sucess = sql """
+ SHOW RESOURCES WHERE NAME = "crete_policy_1_hdfs";
+ """
+ assertEquals(create_sucess.size(), 10)
+
+ def failed_cannot_create_duplicate_name_resources = try_sql """
+ CREATE RESOURCE "crete_policy_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ // errCode = 2, detailMessage = Resource(crete_policy_1) already exist
+ assertEquals(failed_cannot_create_duplicate_name_resources, null)
+
+ // test if we could create one polycy with the same name but
+ // with different properties
+ def failed_cannot_create_duplicate_name_resources_with_different_properties = try_sql """
+ CREATE RESOURCE "crete_policy_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive1",
+ "hadoop.password"="hive1",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ assertEquals(failed_cannot_create_duplicate_name_resources_with_different_properties, null)
+
+ // delete the policy with the same name then try to
+ // create policies with different property each time
+ def drop_result = try_sql """
+ DROP RESOURCE 'crete_policy_1_hdfs'
+ """
+ assertEquals(drop_result.size(), 1)
+
+ def create_duplicate_name_resources_different_username = try_sql """
+ CREATE RESOURCE "crete_policy_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive1",
+ "hadoop.password"="hive1",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ assertEquals(create_duplicate_name_resources_different_username.size(), 1)
+
+ drop_result = try_sql """
+ DROP RESOURCE 'crete_policy_1_hdfs'
+ """
+
+ assertEquals(drop_result.size(), 1)
+
+ def create_duplicate_name_resources_different_dfs_nameservices = try_sql """
+ CREATE RESOURCE "crete_policy_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ assertEquals(create_duplicate_name_resources_different_dfs_nameservices.size(), 1)
+
+ drop_result = try_sql """
+ DROP RESOURCE 'crete_policy_1_hdfs'
+ """
+ assertEquals(drop_result.size(), 1)
+ }
+
+ // can't create success, due to missing required items
+ def has_created_2 = sql """
+ SHOW RESOURCES WHERE NAME = "crete_policy_2_hdfs";
+ """
+ if (has_created_2.size() == 0) {
+ def failed_create_1 = try_sql """
+ CREATE RESOURCE "crete_policy_2_hdfs"
+ PROPERTIES(
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ // errCode = 2, detailMessage = Resource type can't be null
+ assertEquals(failed_create_1, null)
+ }
+
+ if (has_created_2.size() == 0) {
+ def failed_create_2 = try_sql """
+ CREATE RESOURCE "crete_policy_2_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ // errCode = 2, detailMessage = Missing [fs.defaultFS] in properties.
+ assertEquals(failed_create_2, [[0]])
+ }
+
+ if (has_created_2.size() == 0) {
+ def failed_create_2 = try_sql """
+ CREATE RESOURCE "crete_policy_2_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ // errCode = 2, detailMessage = Missing [hadoop.username] in properties.
+ assertEquals(failed_create_2, null)
+ }
+
+ if (has_created_2.size() == 0) {
+ def failed_create_2 = try_sql """
+ CREATE RESOURCE "crete_policy_2_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ // errCode = 2, detailMessage = Missing [dfs.nameservices] in properties.
+ assertEquals(failed_create_2, null)
+ }
+
+ if (has_created_2.size() == 0) {
+ def failed_create_2 = try_sql """
+ CREATE RESOURCE "crete_policy_2_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ // errCode = 2, detailMessage = Missing [dfs.ha.namenodes.my_ha] in properties.
+ assertEquals(failed_create_2, null)
+ }
+
+ // create policy
+
+
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ if (!storage_exist.call("testPolicy_10_hdfs")) {
+ def create_s3_resource = try_sql """
+ CREATE RESOURCE "testPolicy_10_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY testPolicy_10_hdfs
+ PROPERTIES(
+ "storage_resource" = "testPolicy_10_resource_hdfs",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ assertEquals(storage_exist.call("testPolicy_10_hdfs"), true)
+
+ sql"""
+ DROP STORAGE POLICY testPolicy_10_hdfs;
+ """
+
+ sql"""
+ DROP RESOURCE testPolicy_10_resource_hdfs;
+ """
+ }
+
+ if (!storage_exist.call("testPolicy_11_hdfs")) {
+ def create_s3_resource = try_sql """
+ CREATE RESOURCE "testPolicy_11_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY testPolicy_11_hdfs
+ PROPERTIES(
+ "storage_resource" = "testPolicy_11_resource_hdfs",
+ "cooldown_ttl" = "10086"
+ );
+ """
+ assertEquals(storage_exist.call("testPolicy_11_hdfs"), true)
+
+ sql """
+ DROP STORAGE POLICY testPolicy_11_hdfs;
+ """
+
+ sql """
+ DROP RESOURCE "testPolicy_11_resource_hdfs";
+ """
+ }
+
+ if (!storage_exist.call("testPolicy_12_hdfs")) {
+ def create_s3_resource = try_sql """
+ CREATE RESOURCE "testPolicy_12_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY testPolicy_12_hdfs
+ PROPERTIES(
+ "storage_resource" = "testPolicy_12_resource_hdfs",
+ "cooldown_ttl" = "10086",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ // errCode = 2, detailMessage = cooldown_datetime and cooldown_ttl can't be set together.
+ assertEquals(storage_exist.call("testPolicy_12_hdfs"), false)
+
+ sql """
+ DROP RESOURCE "testPolicy_12_resource_hdfs"
+ """
+ }
+
+ if (!storage_exist.call("testPolicy_13_hdfs")) {
+ def create_s3_resource = try_sql """
+ CREATE RESOURCE "testPolicy_13_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY testPolicy_13_hdfs
+ PROPERTIES(
+ "storage_resource" = "testPolicy_13_resource_hdfs",
+ "cooldown_ttl" = "-10086"
+ );
+ """
+ // errCode = 2, detailMessage = cooldown_ttl must >= 0.
+ assertEquals(storage_exist.call("testPolicy_13_hdfs"), false)
+ sql """
+ DROP RESOURCE "testPolicy_13_resource_hdfs"
+ """
+ }
+
+ if (!storage_exist.call("testPolicy_14_hdfs")) {
+ def create_s3_resource = try_sql """
+ CREATE RESOURCE "testPolicy_14_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY testPolicy_14_hdfs
+ PROPERTIES(
+ "storage_resource" = "testPolicy_14_resource_hdfs",
+ "cooldown_datetime" = "2025-06-08"
+ );
+ """
+ // errCode = 2, detailMessage = cooldown_datetime format error: 2022-06-08
+ assertEquals(storage_exist.call("testPolicy_14_hdfs"), false)
+ }
+
+ if (!storage_exist.call("testPolicy_15_hdfs")) {
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY testPolicy_15_hdfs
+ PROPERTIES(
+ "storage_resource" = "s3_resource_not_exist_hdfs",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ // errCode = 2, detailMessage = storage resource doesn't exist: s3_resource_not_exist_hdfs
+ assertEquals(storage_exist.call("testPolicy_15_hdfs"), false)
+ }
+
+ if (!storage_exist.call("testPolicy_redundant_name_hdfs")) {
+ def create_s3_resource = try_sql """
+ CREATE RESOURCE "testPolicy_redundant_name_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_s3_resource_1 = try_sql """
+ CREATE RESOURCE "testPolicy_redundant_name_resource_1_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY testPolicy_redundant_name_hdfs
+ PROPERTIES(
+ "storage_resource" = "testPolicy_redundant_name_resource_hdfs",
+ "cooldown_ttl" = "10086"
+ );
+ """
+ try {
+ sql """
+ CREATE STORAGE POLICY testPolicy_redundant_name_hdfs
+ PROPERTIES(
+ "storage_resource" = "testPolicy_redundant_name_resource_1_hdfs",
+ "cooldown_ttl" = "10086"
+ );
+ """
+ } catch (Exception e) {
+ log.info(e.getMessage())
+ assertTrue(e.getMessage().contains('already create'))
+ }
+ }
+}
+
diff --git a/regression-test/suites/cold_heat_separation/policy/drop_hdfs_reource.groovy b/regression-test/suites/cold_heat_separation/policy/drop_hdfs_reource.groovy
new file mode 100644
index 00000000000..39bbaf09e7a
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/policy/drop_hdfs_reource.groovy
@@ -0,0 +1,148 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("drop_hdfs_policy") {
+ def storage_exist = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(iter in show_storage_policy){
+ if(name == iter[0]){
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def resource_not_table_use = "resource_not_table_use_hdfs"
+ def use_policy = "use_policy_hdfs"
+
+ if (!storage_exist.call(use_policy)) {
+ def has_resouce_policy_drop = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_not_table_use}";
+ """
+ if(has_resouce_policy_drop.size() == 0) {
+ sql """
+ CREATE RESOURCE "${resource_not_table_use}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ }
+
+
+ def drop_result = try_sql """
+ DROP RESOURCE ${resource_not_table_use}
+ """
+ // can drop, no policy use
+ assertEquals(drop_result.size(), 1)
+
+ def resource_table_use = "resource_table_use_hdfs"
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_table_use}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY ${use_policy}
+ PROPERTIES(
+ "storage_resource" = "${resource_table_use}",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ assertEquals(storage_exist.call(use_policy), true)
+
+ def drop_s3_resource_result_1 = try_sql """
+ DROP RESOURCE ${resource_table_use}
+ """
+ // errCode = 2, detailMessage = S3 resource used by policy, can't drop it.
+ assertEquals(drop_s3_resource_result_1, null)
+
+ def drop_policy_ret = try_sql """
+ DROP STORAGE POLICY ${use_policy}
+ """
+ // can drop, no table use
+ assertEquals(drop_policy_ret.size(), 1)
+
+ def create_succ_2 = try_sql """
+ CREATE STORAGE POLICY IF NOT EXISTS drop_policy_test_has_table_binded_hdfs
+ PROPERTIES(
+ "storage_resource" = "${resource_table_use}",
+ "cooldown_datetime" = "2025-06-08 00:00:00"
+ );
+ """
+ assertEquals(storage_exist.call("drop_policy_test_has_table_binded_hdfs"), true)
+
+ // success
+ def create_table_use_created_policy = try_sql """
+ CREATE TABLE IF NOT EXISTS create_table_binding_created_policy_hdfs
+ (
+ k1 BIGINT,
+ k2 LARGEINT,
+ v1 VARCHAR(2048)
+ )
+ UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH (k1) BUCKETS 3
+ PROPERTIES(
+ "storage_policy" = "drop_policy_test_has_table_binded_hdfs",
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "false"
+ );
+ """
+
+ assertEquals(create_table_use_created_policy.size(), 1);
+
+ def drop_policy_fail_ret = try_sql """
+ DROP STORAGE POLICY drop_policy_test_has_table_binded_hdfs
+ """
+ // fail to drop, there are tables using this policy
+ assertEquals(drop_policy_fail_ret, null)
+
+ sql """
+ DROP TABLE create_table_binding_created_policy_hdfs;
+ """
+
+ sql """
+ DROP STORAGE POLICY drop_policy_test_has_table_binded_hdfs;
+ """
+
+ sql """
+ DROP RESOURCE ${resource_table_use};
+ """
+ }
+
+}
+
diff --git a/regression-test/suites/cold_heat_separation/policy/show_hdfs_resource.groovy b/regression-test/suites/cold_heat_separation/policy/show_hdfs_resource.groovy
new file mode 100644
index 00000000000..70e06ea15f1
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation/policy/show_hdfs_resource.groovy
@@ -0,0 +1,70 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("show_hdfs_policy") {
+ def get_storage_policy = { name ->
+ def show_storage_policy = sql """
+ SHOW STORAGE POLICY;
+ """
+ for(policy in show_storage_policy){
+ if(name == policy[0]){
+ return policy;
+ }
+ }
+ return [];
+ }
+
+ if (get_storage_policy.call("showPolicy_1_policy_hdfs").isEmpty()){
+ def create_s3_resource = try_sql """
+ CREATE RESOURCE "showPolicy_1_resource_hdfs"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="127.0.0.1:8120",
+ "hadoop.username"="hive",
+ "hadoop.password"="hive",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+ def create_succ_1 = try_sql """
+ CREATE STORAGE POLICY showPolicy_1_policy_hdfs
+ PROPERTIES(
+ "storage_resource" = "showPolicy_1_resource_hdfs",
+ "cooldown_datetime" = "2022-06-08 00:00:00"
+ );
+ """
+ }
+ def show_result = sql """
+ SHOW STORAGE POLICY;
+ """
+
+ def storage_policy = get_storage_policy("showPolicy_1_policy_hdfs")
+ assertEquals(storage_policy.size(), 7)
+ assertEquals(storage_policy[4], "showPolicy_1_resource_hdfs")
+ assertEquals(storage_policy[5], "2022-06-08 00:00:00")
+ sql """
+ DROP STORAGE POLICY showPolicy_1_policy_hdfs;
+ """
+
+ sql """
+ DROP RESOURCE "showPolicy_1_resource_hdfs";
+ """
+}
+
diff --git a/regression-test/suites/cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy
new file mode 100644
index 00000000000..3437bcfc37b
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation_p2/add_drop_partition_by_hdfs.groovy
@@ -0,0 +1,284 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import java.time.LocalDate;
+
+suite("add_drop_partition_by_hdfs") {
+ def fetchBeHttp = { check_func, meta_url ->
+ def i = meta_url.indexOf("/api")
+ String endPoint = meta_url.substring(0, i)
+ String metaUri = meta_url.substring(i)
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+ httpTest {
+ endpoint endPoint
+ uri metaUri
+ op "get"
+ check check_func
+ }
+ }
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = { data_sizes, t ->
+ def tabletId = t[0]
+ String meta_url = t[17]
+ def clos = { respCode, body ->
+ logger.info("test ttl expired resp Code {}", "${respCode}".toString())
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
+ }
+ // used as passing out parameter to fetchDataSize
+ List<Long> sizes = [-1, -1]
+ def tableName = "tbl1"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+
+ def check_storage_policy_exist = { name->
+ def polices = sql"""
+ show storage policy;
+ """
+ for (p in polices) {
+ if (name == p[0]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def resource_name = "test_add_drop_partition_resource"
+ def policy_name= "test_add_drop_partition_policy"
+
+ if (check_storage_policy_exist(policy_name)) {
+ sql """
+ DROP STORAGE POLICY ${policy_name}
+ """
+ }
+
+ def has_resouce = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ if (has_resouce.size() > 0) {
+ sql """
+ DROP RESOURCE ${resource_name}
+ """
+ }
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policy_name}
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "300"
+ )
+ """
+
+ // test one replica
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE ${tableName} (
+ `k1` int,
+ `k2` date
+ )
+ PARTITION BY RANGE(k2)(
+ partition p1 VALUES LESS THAN ("2014-01-01"),
+ partition p2 VALUES LESS THAN ("2015-01-01"),
+ partition p3 VALUES LESS THAN ("2016-01-01")
+ )
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES
+ (
+ "replication_num" = "1",
+ "storage_policy" = "${policy_name}"
+ );
+ """
+ sql """
+ insert into ${tableName} values(1, "2013-01-01");
+ """
+ sql """
+ insert into ${tableName} values(1, "2014-01-01");
+ """
+ sql """
+ insert into ${tableName} values(1, "2015-01-01");
+ """
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ def LocalDataSize1 = sizes[0]
+ def RemoteDataSize1 = sizes[1]
+ log.info( "test local size {} not zero, remote size {}", LocalDataSize1, RemoteDataSize1)
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+ def originLocalDataSize1 = LocalDataSize1;
+
+ // 等待10min，show tablets from table, 预期not_use_storage_policy_tablet_list 的 RemoteDataSize 为0，LocalDataSize不为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ Long sleepTimes = 0;
+ while (RemoteDataSize1 != originLocalDataSize1 && sleepTimes < 60) {
+ log.info( "test remote size is same with origin size, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ sleepTimes += 1
+ }
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
+ log.info( "test remote size not zero")
+ assertEquals(RemoteDataSize1, originLocalDataSize1)
+
+ // 12列是storage policy
+ def partitions = sql "show partitions from ${tableName}"
+ for (par in partitions) {
+ assertTrue(par[12] == "${policy_name}")
+ }
+
+ try_sql """
+ drop storage policy add_policy;
+ """
+
+ try_sql """
+ drop resource add_resource;
+ """
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "add_resource"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ try_sql """
+ create storage policy tmp_policy
+ PROPERTIES( "storage_resource" = "add_resource", "cooldown_ttl" = "300");
+ """
+
+ // can not set to one policy with different resource
+ try {
+ sql """alter table ${tableName} set ("storage_policy" = "add_policy");"""
+ } catch (java.sql.SQLException t) {
+ assertTrue(true)
+ }
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS add_policy1
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "60"
+ )
+ """
+
+ sql """alter table ${tableName} set ("storage_policy" = "add_policy1");"""
+
+ // wait for report
+ sleep(300000)
+
+ partitions = sql "show partitions from ${tableName}"
+ for (par in partitions) {
+ assertTrue(par[12] == "add_policy1")
+ }
+
+
+ sql """
+ alter table ${tableName} ADD PARTITION np
+ VALUES LESS THAN ("2017-01-01");
+ """
+
+ sql """
+ insert into ${tableName} values(1, "2017-01-01");
+ """
+
+ partitions = sql "show partitions from ${tableName}"
+ for (par in partitions) {
+ assertTrue(par[12] == "add_policy1")
+ }
+
+ sql """
+ select * from ${tableName}
+ """
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+ sql """
+ drop storage policy add_policy;
+ """
+
+ sql """
+ drop storage policy add_policy1;
+ """
+
+ sql """
+ drop resource add_resource;
+ """
+
+
+
+}
diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_dynamic_partition_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_dynamic_partition_by_hdfs.groovy
new file mode 100644
index 00000000000..a007b26305d
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_dynamic_partition_by_hdfs.groovy
@@ -0,0 +1,279 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import java.time.LocalDate;
+
+suite("cold_heat_dynamic_partition_by_hdfs") {
+ def fetchBeHttp = { check_func, meta_url ->
+ def i = meta_url.indexOf("/api")
+ String endPoint = meta_url.substring(0, i)
+ String metaUri = meta_url.substring(i)
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+ httpTest {
+ endpoint endPoint
+ uri metaUri
+ op "get"
+ check check_func
+ }
+ }
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = { data_sizes, t ->
+ def tabletId = t[0]
+ String meta_url = t[17]
+ def clos = { respCode, body ->
+ logger.info("test ttl expired resp Code {}", "${respCode}".toString())
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
+ }
+ // used as passing out parameter to fetchDataSize
+ List<Long> sizes = [-1, -1]
+ def tableName = "tbl2"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+
+ def check_storage_policy_exist = { name->
+ def polices = sql"""
+ show storage policy;
+ """
+ for (p in polices) {
+ if (name == p[0]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def resource_name = "test_dynamic_partition_resource"
+ def policy_name= "test_dynamic_partition_policy"
+
+ if (check_storage_policy_exist(policy_name)) {
+ sql """
+ DROP STORAGE POLICY ${policy_name}
+ """
+ }
+
+ def has_resouce = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ if (has_resouce.size() > 0) {
+ sql """
+ DROP RESOURCE IF EXISTS ${resource_name}
+ """
+ }
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policy_name}
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "300"
+ )
+ """
+
+ // test one replica
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE ${tableName} (
+ `k1` int,
+ `k2` date
+ )
+ PARTITION BY RANGE(k2)()
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES
+ (
+ "dynamic_partition.enable" = "true",
+ "dynamic_partition.time_unit" = "DAY",
+ "dynamic_partition.end" = "3",
+ "dynamic_partition.prefix" = "p",
+ "dynamic_partition.buckets" = "1",
+ "dynamic_partition.replication_num" = "1",
+ "dynamic_partition.start" = "-3",
+ "storage_policy" = "${policy_name}",
+ "replication_num" = "1"
+ );
+ """
+ LocalDate currentDate = LocalDate.now();
+ LocalDate currentDatePlusOne = currentDate.plusDays(1);
+ LocalDate currentDatePlusTwo = currentDate.plusDays(2);
+ LocalDate currentDatePlusThree = currentDate.plusDays(3);
+ sql """
+ insert into ${tableName} values(1, "${currentDate.toString()}");
+ """
+ sql """
+ insert into ${tableName} values(1, "${currentDatePlusOne.toString()}");
+ """
+ sql """
+ insert into ${tableName} values(1, "${currentDatePlusTwo.toString()}");
+ """
+ sql """
+ insert into ${tableName} values(1, "${currentDatePlusThree.toString()}");
+ """
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ def LocalDataSize1 = sizes[0]
+ def RemoteDataSize1 = sizes[1]
+ log.info( "test local size {} not zero, remote size {}", LocalDataSize1, RemoteDataSize1)
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+ def originLocalDataSize1 = LocalDataSize1;
+
+ // 等待10min，show tablets from table, 预期not_use_storage_policy_tablet_list 的 RemoteDataSize 为0，LocalDataSize不为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ Long sleepTimes = 0;
+ while (RemoteDataSize1 != originLocalDataSize1 && sleepTimes < 60) {
+ log.info( "test remote size is same with origin size, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM
+ """
+ fetchDataSize(sizes, tablets[0])
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ sleepTimes += 1
+ }
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
+ log.info( "test remote size not zero")
+ assertEquals(RemoteDataSize1, originLocalDataSize1)
+
+ // 12列是storage policy
+ def partitions = sql "show partitions from ${tableName}"
+ for (par in partitions) {
+ assertTrue(par[12] == "${policy_name}")
+ }
+
+ try_sql """
+ drop storage policy tmp_policy;
+ """
+
+ try_sql """
+ drop resource tmp_resource;
+ """
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "tmp_resource"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ try_sql """
+ create storage policy tmp_policy
+ PROPERTIES( "storage_resource" = "tmp_resource", "cooldown_ttl" = "300");
+ """
+
+ // can not set to one policy with different resource
+ try {
+ sql """alter table ${tableName} set ("storage_policy" = "tmp_policy");"""
+ } catch (java.sql.SQLException t) {
+ assertTrue(true)
+ }
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS tmp_policy1
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "60"
+ )
+ """
+
+ sql """alter table ${tableName} set ("storage_policy" = "tmp_policy1");"""
+
+ // wait for report
+ sleep(300000)
+
+ partitions = sql "show partitions from ${tableName}"
+ for (par in partitions) {
+ assertTrue(par[12] == "tmp_policy1")
+ }
+
+ sql """
+ select * from ${tableName}
+ """
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+ sql """
+ drop storage policy tmp_policy;
+ """
+
+ sql """
+ drop storage policy tmp_policy1;
+ """
+
+ sql """
+ drop resource tmp_resource;
+ """
+
+
+
+}
diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy_by_hdfs.groovy
new file mode 100644
index 00000000000..723f50721ec
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_partition_policy_by_hdfs.groovy
@@ -0,0 +1,370 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("create_table_use_partition_policy_by_hdfs") {
+ def fetchBeHttp = { check_func, meta_url ->
+ def i = meta_url.indexOf("/api")
+ String endPoint = meta_url.substring(0, i)
+ String metaUri = meta_url.substring(i)
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+ httpTest {
+ endpoint endPoint
+ uri metaUri
+ op "get"
+ check check_func
+ }
+ }
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = { data_sizes, t ->
+ def tabletId = t[0]
+ String meta_url = t[17]
+ def clos = { respCode, body ->
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ logger.info("test ttl expired resp Code {}, body {}", "${respCode}".toString(), out)
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
+ }
+ // used as passing out parameter to fetchDataSize
+ List<Long> sizes = [-1, -1]
+ def tableName = "lineitem1"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ def stream_load_one_part = { partnum ->
+ streamLoad {
+ table tableName
+ // a default db 'regression_test' is specified in
+ // ${DORIS_HOME}/conf/regression-conf.groovy
+
+ // default label is UUID:
+ // set 'label' UUID.randomUUID().toString()
+
+ // default column_separator is specify in doris fe config, usually is '\t'.
+ // this line change to ','
+ set 'column_separator', '|'
+ set 'compress_type', 'GZ'
+
+
+ // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+ // also, you can stream load a http stream, e.g. http://xxx/some.csv
+ file """${getS3Url()}/regression/tpch/sf1/lineitem.csv.split${partnum}.gz"""
+
+ time 10000 // limit inflight 10s
+
+ // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+
+ // if declared a check callback, the default check condition will ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+ }
+
+ def load_lineitem_table = {
+ stream_load_one_part("00")
+ stream_load_one_part("01")
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ while (tablets[0][8] == "0") {
+ log.info( "test local size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ }
+ }
+
+ def check_storage_policy_exist = { name->
+ def polices = sql"""
+ show storage policy;
+ """
+ for (p in polices) {
+ if (name == p[0]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def resource_name = "test_table_partition_with_data_resource"
+ def policy_name= "test_table_partition_with_data_policy"
+
+ if (check_storage_policy_exist(policy_name)) {
+ sql """
+ DROP STORAGE POLICY ${policy_name}
+ """
+ }
+
+ def has_resouce = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ if (has_resouce.size() > 0) {
+ sql """
+ DROP RESOURCE ${resource_name}
+ """
+ }
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policy_name}
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "300"
+ )
+ """
+
+ // test one replica
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ PARTITION BY RANGE(`L_SHIPDATE`)
+ (
+ PARTITION `p202301` VALUES LESS THAN ("1995-12-01") ("storage_policy" = "${policy_name}"),
+ PARTITION `p202302` VALUES LESS THAN ("2017-03-01")
+ )
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1"
+ )
+ """
+
+ load_lineitem_table()
+ // sleep(30000);
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ def LocalDataSize1 = sizes[0]
+ def RemoteDataSize1 = sizes[1]
+ log.info( "test local size {} not zero, remote size {}", LocalDataSize1, RemoteDataSize1)
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+ def originLocalDataSize1 = LocalDataSize1;
+
+ // 等待10min，show tablets from table, 预期not_use_storage_policy_tablet_list 的 RemoteDataSize 为0，LocalDataSize不为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ Long sleepTimes = 0;
+ while (RemoteDataSize1 != originLocalDataSize1 && sleepTimes < 60) {
+ log.info( "test remote size is same with origin size, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ fetchDataSize(sizes, tablets[0])
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ sleepTimes += 1
+ }
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
+ log.info( "test remote size not zero")
+ assertEquals(RemoteDataSize1, originLocalDataSize1)
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202302)
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ log.info( "test local size is not zero")
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+ // test table with 3 replication
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ PARTITION BY RANGE(`L_SHIPDATE`)
+ (
+ PARTITION `p202301` VALUES LESS THAN ("1995-12-01") ("storage_policy" = "${policy_name}"),
+ PARTITION `p202302` VALUES LESS THAN ("2017-03-01")
+ )
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "3"
+ )
+ """
+
+ load_lineitem_table()
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ log.info( "test local size not zero")
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+ originLocalDataSize1 = LocalDataSize1
+
+ // 等待10min，show tablets from table, 预期not_use_storage_policy_tablet_list 的 RemoteDataSize 为0，LocalDataSize不为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ sleepTimes = 0
+ while (RemoteDataSize1 != originLocalDataSize1 && sleepTimes < 60) {
+ log.info( "test remote size is same with origin size, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202301)
+ """
+ fetchDataSize(sizes, tablets[0])
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ sleepTimes += 1
+ }
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
+ log.info( "test remote size not zero")
+ assertEquals(RemoteDataSize1, originLocalDataSize1)
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202302)
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ log.info( "test local size is not zero")
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+
+
+}
diff --git a/regression-test/suites/cold_heat_separation_p2/create_table_use_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/create_table_use_policy_by_hdfs.groovy
new file mode 100644
index 00000000000..c80f82a167d
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation_p2/create_table_use_policy_by_hdfs.groovy
@@ -0,0 +1,294 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("create_table_use_policy_by_hdfs") {
+ def fetchBeHttp = { check_func, meta_url ->
+ def i = meta_url.indexOf("/api")
+ String endPoint = meta_url.substring(0, i)
+ String metaUri = meta_url.substring(i)
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+ httpTest {
+ endpoint endPoint
+ uri metaUri
+ op "get"
+ check check_func
+ }
+ }
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = { data_sizes, t ->
+ def tabletId = t[0]
+ String meta_url = t[17]
+ def clos = { respCode, body ->
+ logger.info("test ttl expired resp Code {}", "${respCode}".toString())
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
+ }
+ // used as passing out parameter to fetchDataSize
+ List<Long> sizes = [-1, -1]
+ def tableName = "lineitem2"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ def stream_load_one_part = { partnum ->
+ streamLoad {
+ table tableName
+ // a default db 'regression_test' is specified in
+ // ${DORIS_HOME}/conf/regression-conf.groovy
+
+ // default label is UUID:
+ // set 'label' UUID.randomUUID().toString()
+
+ // default column_separator is specify in doris fe config, usually is '\t'.
+ // this line change to ','
+ set 'column_separator', '|'
+ set 'compress_type', 'GZ'
+
+
+ // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+ // also, you can stream load a http stream, e.g. http://xxx/some.csv
+ file """${getS3Url()}/regression/tpch/sf1/lineitem.csv.split${partnum}.gz"""
+
+ time 10000 // limit inflight 10s
+
+ // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+
+ // if declared a check callback, the default check condition will ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+ }
+
+ def load_lineitem_table = {
+ stream_load_one_part("00")
+ stream_load_one_part("01")
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ while (tablets[0][8] == "0") {
+ log.info( "test local size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ }
+ }
+
+ def check_storage_policy_exist = { name->
+ def polices = sql"""
+ show storage policy;
+ """
+ for (p in polices) {
+ if (name == p[0]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def resource_name = "test_table_with_data_resource"
+ def policy_name= "test_table_with_data_policy"
+
+ if (check_storage_policy_exist(policy_name)) {
+ sql """
+ DROP STORAGE POLICY ${policy_name}
+ """
+ }
+
+ def has_resouce = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ if (has_resouce.size() > 0) {
+ sql """
+ DROP RESOURCE ${resource_name}
+ """
+ }
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policy_name}
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "300"
+ )
+ """
+
+ // test one replica
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_policy" = "${policy_name}"
+ )
+ """
+
+ load_lineitem_table()
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ def LocalDataSize1 = sizes[0]
+
+ // 等待10min，show tablets from table, 预期RemoteDataSize 不为0，且等于LocalDataSize1，预期LocalDataSize 为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(tablets.size() > 0)
+ log.info( "test remote size not zero")
+ assertEquals(LocalDataSize1, sizes[1])
+ log.info( "test local size is zero")
+ assertEquals(0, sizes[0])
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+ // test table with 3 replication
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "3",
+ "storage_policy" = "${policy_name}"
+ )
+ """
+
+ load_lineitem_table()
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ LocalDataSize1 = sizes[0]
+
+ // 等待10min，show tablets from table, 预期RemoteDataSize 不为0，且等于LocalDataSize1，预期LocalDataSize 为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(tablets.size() > 0)
+ log.info( "test remote size not zero")
+ assertEquals(LocalDataSize1, sizes[1])
+ log.info( "test local size is zero")
+ assertEquals(0, sizes[0])
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+
+
+}
diff --git a/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition_by_hdfs.groovy
new file mode 100644
index 00000000000..b1371788de4
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation_p2/modify_replica_use_partition_by_hdfs.groovy
@@ -0,0 +1,506 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("modify_replica_use_partition_by_hdfs") {
+ def fetchBeHttp = { check_func, meta_url ->
+ def i = meta_url.indexOf("/api")
+ String endPoint = meta_url.substring(0, i)
+ String metaUri = meta_url.substring(i)
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+ httpTest {
+ endpoint endPoint
+ uri metaUri
+ op "get"
+ check check_func
+ }
+ }
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = { data_sizes, t ->
+ def tabletId = t[0]
+ String meta_url = t[17]
+ def clos = { respCode, body ->
+ logger.info("test ttl expired resp Code {}", "${respCode}".toString())
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
+ }
+ // used as passing out parameter to fetchDataSize
+ List<Long> sizes = [-1, -1]
+ def get_meta = { url ->
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl ")
+ sb.append(url)
+ String command = sb.toString()
+ log.info(command)
+ def process = command.execute()
+ int code = process.waitFor()
+ def err = IOGroovyMethods.getText(new BufferedReader(new InputStreamReader(process.getErrorStream())));
+ def out = process.getText()
+ logger.info("Run compaction: code=" + code + ", err=" + err)
+ assertEquals(code, 0)
+ return out
+ }
+
+ def tableName = "lineitem3"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ def stream_load_one_part = { partnum ->
+ streamLoad {
+ table tableName
+ // a default db 'regression_test' is specified in
+ // ${DORIS_HOME}/conf/regression-conf.groovy
+
+ // default label is UUID:
+ // set 'label' UUID.randomUUID().toString()
+
+ // default column_separator is specify in doris fe config, usually is '\t'.
+ // this line change to ','
+ set 'column_separator', '|'
+ set 'compress_type', 'GZ'
+
+
+ // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+ // also, you can stream load a http stream, e.g. http://xxx/some.csv
+ file """${getS3Url()}/regression/tpch/sf1/lineitem.csv.split${partnum}.gz"""
+
+ time 10000 // limit inflight 10s
+
+ // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+
+ // if declared a check callback, the default check condition will ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+ }
+
+ def load_lineitem_table = {
+ stream_load_one_part("00")
+ stream_load_one_part("01")
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ while (tablets[0][8] == "0") {
+ log.info( "test local size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ }
+ }
+
+ def check_storage_policy_exist = { name->
+ def polices = sql"""
+ show storage policy;
+ """
+ for (p in polices) {
+ if (name == p[0]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def resource_name = "test_table_replica_with_data_resource"
+ def policy_name= "test_table_replica_with_data_policy"
+
+ if (check_storage_policy_exist(policy_name)) {
+ sql """
+ DROP STORAGE POLICY ${policy_name}
+ """
+ }
+
+ def has_resouce = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ if (has_resouce.size() > 0) {
+ sql """
+ DROP RESOURCE ${resource_name}
+ """
+ }
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policy_name}
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "300"
+ )
+ """
+
+ // test one replica then we will alter all the replica of all partitions to 3
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ PARTITION BY RANGE(`L_SHIPDATE`)
+ (
+ PARTITION `p202301` VALUES LESS THAN ("1995-12-01"),
+ PARTITION `p202302` VALUES LESS THAN ("2017-03-01")
+ )
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "storage_policy" = "${policy_name}",
+ "replication_num" = "1"
+ )
+ """
+
+ load_lineitem_table()
+
+ // 等待10min，show tablets from table, 预期not_use_storage_policy_tablet_list 的 RemoteDataSize 为LocalDataSize1，LocalDataSize为0
+ sleep(600000)
+
+
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ assertTrue(tablets.size() > 0)
+ def LocalDataSize1 = sizes[0]
+ def RemoteDataSize1 = sizes[1]
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
+ log.info( "test remote size not zero")
+ assertTrue(RemoteDataSize1 != 0)
+ def originSize = tablets.size()
+
+ // alter change replication num
+ sql """
+ ALTER TABLE ${tableName}
+ MODIFY PARTITION (p202301, p202302) SET("replication_num"="3");
+ """
+
+ sql """
+ select count(*) from ${tableName}
+ """
+
+ sql """
+ select * from ${tableName} limit 10
+ """
+ // wait one minute for migration to be completed
+ sleep(60000)
+
+ // 对比所有tablets的replicas的rowsets meta是否相同
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ while (tablets.size() != 3 * originSize) {
+ log.info( "tablets clone not finished, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ }
+ def compactionStatusIdx = tablets[0].size() - 1
+ // check rowsets inside the 3 replica
+ def iterate_num = tablets.size() / 3;
+ for (int i = 0; i < iterate_num; i++) {
+ int idx = i * 3;
+ def dst = tablets[idx][18]
+ def text = get_meta(dst)
+ def obj = new JsonSlurper().parseText(text)
+ def rowsets = obj.rowsets
+ for (x in [1,2]) {
+ dst = tablets[idx + x][18]
+ text = get_meta(dst)
+ obj = new JsonSlurper().parseText(text)
+ log.info( "test rowset meta is the same")
+ assertEquals(rowsets, obj.rowsets)
+ }
+ }
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+ // test table with 3 replication, then we will alter all the replica of all partitions to 1
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ PARTITION BY RANGE(`L_SHIPDATE`)
+ (
+ PARTITION `p202301` VALUES LESS THAN ("1995-12-01"),
+ PARTITION `p202302` VALUES LESS THAN ("2017-03-01")
+ )
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "storage_policy" = "${policy_name}",
+ "replication_num" = "3"
+ )
+ """
+
+ load_lineitem_table()
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ log.info( "test local size not zero")
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+
+ // 等待10min，show tablets from table, 预期not_use_storage_policy_tablet_list 的 RemoteDataSize 为LocalDataSize1，LocalDataSize为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
+ log.info( "test remote size not zero")
+ assertTrue(RemoteDataSize1 != 0)
+
+ // alter change replication num
+ sql """
+ ALTER TABLE ${tableName}
+ MODIFY PARTITION (p202301, p202302) SET("replication_num"="1");
+ """
+
+ sql """
+ select count(*) from ${tableName}
+ """
+
+ sql """
+ select * from ${tableName} limit 10
+ """
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+
+ // test table with one partition p1 3 replication and another p2 with 1 replication,
+ // then we will alter p1 to one replica and p2 to 3
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ PARTITION BY RANGE(`L_SHIPDATE`)
+ (
+ PARTITION `p202301` VALUES LESS THAN ("1995-12-01") ("replication_num" = "3"),
+ PARTITION `p202302` VALUES LESS THAN ("2017-03-01") ("replication_num" = "1")
+ )
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "storage_policy" = "${policy_name}"
+ )
+ """
+
+ load_lineitem_table()
+
+ // show tablets from table, 获取第一个tablet的 LocalDataSize1
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ log.info( "test local size not zero")
+ assertTrue(LocalDataSize1 != 0)
+ log.info( "test remote size is zero")
+ assertEquals(RemoteDataSize1, 0)
+
+ // 等待10min，show tablets from table, 预期not_use_storage_policy_tablet_list 的 RemoteDataSize 为LocalDataSize1，LocalDataSize为0
+ sleep(600000)
+
+
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ while (sizes[1] == 0) {
+ log.info( "test remote size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ }
+ LocalDataSize1 = sizes[0]
+ RemoteDataSize1 = sizes[1]
+ log.info( "test local size is zero")
+ assertEquals(LocalDataSize1, 0)
+ log.info( "test remote size not zero")
+ assertTrue(RemoteDataSize1 != 0)
+
+ // alter change replication num
+ sql """
+ ALTER TABLE ${tableName}
+ MODIFY PARTITION (p202301) SET("replication_num"="1");
+ """
+
+ sql """
+ ALTER TABLE ${tableName}
+ MODIFY PARTITION (p202302) SET("replication_num"="3");
+ """
+
+ sql """
+ select count(*) from ${tableName}
+ """
+
+ sql """
+ select * from ${tableName} limit 10
+ """
+
+ // wait one minute for migration to be completed
+ sleep(60000)
+ // 对比3副本的partition中所有tablets的replicas的rowsets meta是否相同
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202302)
+ """
+ // sleep to wait for the report
+ sleep(15000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName} PARTITIONS(p202302)
+ """
+ compactionStatusIdx = tablets[0].size() - 1
+ // check rowsets inside the 3 replica
+ iterate_num = tablets.size() / 3;
+ for (int i = 0; i < iterate_num; i++) {
+ int idx = i * 3;
+ def dst = tablets[idx][18]
+ def text = get_meta(dst)
+ def obj = new JsonSlurper().parseText(text)
+ def rowsets = obj.rowsets
+ for (x in [1,2]) {
+ dst = tablets[idx + x][18]
+ text = get_meta(dst)
+ obj = new JsonSlurper().parseText(text)
+ log.info( "test rowset meta is the same")
+ assertEquals(rowsets, obj.rowsets)
+ }
+ }
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+
+}
diff --git a/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy_by_hdfs.groovy b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy_by_hdfs.groovy
new file mode 100644
index 00000000000..2c3f2d18219
--- /dev/null
+++ b/regression-test/suites/cold_heat_separation_p2/table_modify_resouce_and_policy_by_hdfs.groovy
@@ -0,0 +1,326 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import groovy.json.JsonSlurper
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("table_modify_resouce_by_hdfs") {
+ def fetchBeHttp = { check_func, meta_url ->
+ def i = meta_url.indexOf("/api")
+ String endPoint = meta_url.substring(0, i)
+ String metaUri = meta_url.substring(i)
+ i = endPoint.lastIndexOf('/')
+ endPoint = endPoint.substring(i + 1)
+ httpTest {
+ endpoint endPoint
+ uri metaUri
+ op "get"
+ check check_func
+ }
+ }
+ // data_sizes is one arrayList<Long>, t is tablet
+ def fetchDataSize = { data_sizes, t ->
+ def tabletId = t[0]
+ String meta_url = t[17]
+ def clos = { respCode, body ->
+ logger.info("test ttl expired resp Code {}", "${respCode}".toString())
+ assertEquals("${respCode}".toString(), "200")
+ String out = "${body}".toString()
+ def obj = new JsonSlurper().parseText(out)
+ data_sizes[0] = obj.local_data_size
+ data_sizes[1] = obj.remote_data_size
+ }
+ fetchBeHttp(clos, meta_url.replace("header", "data_size"))
+ }
+ // used as passing out parameter to fetchDataSize
+ List<Long> sizes = [-1, -1]
+ def tableName = "lineitem4"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ def stream_load_one_part = { partnum ->
+ streamLoad {
+ table tableName
+ // a default db 'regression_test' is specified in
+ // ${DORIS_HOME}/conf/regression-conf.groovy
+
+ // default label is UUID:
+ // set 'label' UUID.randomUUID().toString()
+
+ // default column_separator is specify in doris fe config, usually is '\t'.
+ // this line change to ','
+ set 'column_separator', '|'
+ set 'compress_type', 'GZ'
+
+
+ // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv.
+ // also, you can stream load a http stream, e.g. http://xxx/some.csv
+ file """${getS3Url()}/regression/tpch/sf1/lineitem.csv.split${partnum}.gz"""
+
+ time 10000 // limit inflight 10s
+
+ // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows
+
+ // if declared a check callback, the default check condition will ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+ }
+
+ def load_lineitem_table = {
+ stream_load_one_part("00")
+ stream_load_one_part("01")
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ while (tablets[0][8] == "0") {
+ log.info( "test local size is zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ }
+ }
+
+ def check_storage_policy_exist = { name->
+ def polices = sql"""
+ show storage policy;
+ """
+ for (p in polices) {
+ if (name == p[0]) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ def resource_name = "test_table_with_data_resource_modify_1"
+ def policy_name= "test_table_with_data_policy_modify_1"
+
+ if (check_storage_policy_exist(policy_name)) {
+ sql """
+ DROP STORAGE POLICY ${policy_name}
+ """
+ }
+
+ def has_resouce = sql """
+ SHOW RESOURCES WHERE NAME = "${resource_name}";
+ """
+ if (has_resouce.size() > 0) {
+ sql """
+ DROP RESOURCE ${resource_name}
+ """
+ }
+
+ sql """
+ CREATE RESOURCE IF NOT EXISTS "${resource_name}"
+ PROPERTIES(
+ "type"="hdfs",
+ "fs.defaultFS"="${getHdfsFs()}",
+ "hadoop.username"="${getHdfsUser()}",
+ "hadoop.password"="${getHdfsPasswd()}",
+ "dfs.nameservices" = "my_ha",
+ "dfs.ha.namenodes.my_ha" = "my_namenode1, my_namenode2",
+ "dfs.namenode.rpc-address.my_ha.my_namenode1" = "127.0.0.1:10000",
+ "dfs.namenode.rpc-address.my_ha.my_namenode2" = "127.0.0.1:10000",
+ "dfs.client.failover.proxy.provider" = "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+ """
+
+ sql """
+ CREATE STORAGE POLICY IF NOT EXISTS ${policy_name}
+ PROPERTIES(
+ "storage_resource" = "${resource_name}",
+ "cooldown_ttl" = "300"
+ )
+ """
+
+ // test one replica
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "1",
+ "storage_policy" = "${policy_name}"
+ )
+ """
+
+ load_lineitem_table()
+
+ // 等待10分钟 获取remote data size
+ sleep(600000)
+ def tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+
+ def try_times = 100
+ while (sizes[0] != 0) {
+ log.info( "test local size is not zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ try_times -= 1
+ assertTrue(try_times > 0)
+ }
+
+ // 修改resource和policy到新值然后查看remote data size是否能对上
+ sql """
+ ALTER STORAGE POLICY ${policy_name} PROPERTIES("cooldown_ttl" = "500");
+ """
+
+ sql """
+ select count(*) from ${tableName}
+ """
+
+ sql """
+ select max(L_SUPPKEY) from ${tableName}
+ """
+
+ sql """
+ select * from ${tableName} limit 10
+ """
+
+
+ def tablets2 = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ // [8] local data size, [9] remote data size
+ log.info( "test all remote size not zero")
+ for (int i = 0; i < tablets2.size(); i++) {
+ fetchDataSize(sizes, tablets2[i])
+ assertTrue(sizes[1] > 0)
+ }
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+ // test three replica
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL
+ )
+ DUPLICATE KEY(L_ORDERKEY, L_PARTKEY, L_SUPPKEY, L_LINENUMBER)
+ DISTRIBUTED BY HASH(L_ORDERKEY) BUCKETS 3
+ PROPERTIES (
+ "replication_num" = "3",
+ "storage_policy" = "${policy_name}"
+ )
+ """
+
+ load_lineitem_table()
+
+ // 等待10分钟 获取remote data size
+ sleep(600000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ log.info( "test tablets not empty")
+ assertTrue(tablets.size() > 0)
+ fetchDataSize(sizes, tablets[0])
+ try_times = 100
+ while (sizes[0] != 0) {
+ log.info( "test local size is not zero, sleep 10s")
+ sleep(10000)
+ tablets = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ fetchDataSize(sizes, tablets[0])
+ try_times -= 1
+ assertTrue(try_times > 0)
+ }
+
+ // 修改resource和policy到新值然后查看remote data size是否能对上
+ sql """
+ ALTER STORAGE POLICY ${policy_name} PROPERTIES("cooldown_ttl" = "500");
+ """
+
+ sql """
+ select count(*) from ${tableName}
+ """
+
+ sql """
+ select max(L_SUPPKEY) from ${tableName}
+ """
+
+ sql """
+ select * from ${tableName} limit 10
+ """
+
+
+ tablets2 = sql """
+ SHOW TABLETS FROM ${tableName}
+ """
+ // [8] local data size, [9] remote data size
+ log.info( "test all remote size not zero")
+ for (int i = 0; i < tablets2.size(); i++) {
+ fetchDataSize(sizes, tablets2[i])
+ assertTrue(sizes[1] > 0)
+ }
+
+
+ sql """
+ DROP TABLE ${tableName}
+ """
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org