You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/01/04 14:24:57 UTC
[doris] branch master updated: [improvement](storage-policy) Add check validity when create storage policy. (#14405)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 61d538c713 [improvement](storage-policy) Add check validity when create storage policy. (#14405)
61d538c713 is described below
commit 61d538c71315e35115aedab08ddeb7030c776884
Author: deardeng <56...@qq.com>
AuthorDate: Wed Jan 4 22:24:49 2023 +0800
[improvement](storage-policy) Add check validity when create storage policy. (#14405)
---
.../main/java/org/apache/doris/alter/Alter.java | 7 +--
.../java/org/apache/doris/backup/S3Storage.java | 11 ++--
.../java/org/apache/doris/catalog/S3Resource.java | 48 +++++++++++++++++
.../java/org/apache/doris/alter/AlterTest.java | 4 +-
.../org/apache/doris/catalog/ResourceMgrTest.java | 1 +
.../org/apache/doris/catalog/S3ResourceTest.java | 3 ++
.../cold_heat_separation/policy/alter.groovy | 11 ++--
.../cold_heat_separation/policy/create.groovy | 63 +++++++++++++---------
.../suites/cold_heat_separation/policy/drop.groovy | 10 ++--
.../suites/cold_heat_separation/policy/show.groovy | 5 +-
.../use_policy/alter_table_add_policy.groovy | 10 ++--
.../create_table_use_partition_policy.groovy | 10 ++--
.../use_policy/create_table_use_policy.groovy | 5 +-
.../use_policy/modify_partition_add_policy.groovy | 11 ++--
.../use_policy/use_default_storage_policy.groovy | 5 +-
15 files changed, 136 insertions(+), 68 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index d3f188c46e..4c91900423 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -753,12 +753,7 @@ public class Alter {
modifiedProperties.putAll(properties);
// 4.3 modify partition storage policy
- String partitionStoragePolicy = partitionInfo.getStoragePolicy(partition.getId());
- if (!partitionStoragePolicy.equals("")) {
- throw new DdlException("Do not support alter table's partition storage policy , this table ["
- + olapTable.getName() + "] and partition [" + partitionName
- + "] has storage policy " + partitionStoragePolicy);
- }
+ // can set multi times storage policy
String currentStoragePolicy = PropertyAnalyzer.analyzeStoragePolicy(properties);
if (!currentStoragePolicy.equals("")) {
// check currentStoragePolicy resource exist.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java b/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java
index d0828ca87e..3c88ad5dd2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/S3Storage.java
@@ -253,11 +253,14 @@ public class S3Storage extends BlobStorage {
LOG.info("upload content success: " + response.eTag());
return Status.OK;
} catch (S3Exception e) {
- LOG.error("write content failed:", e);
+ LOG.warn("write content failed:", e);
return new Status(Status.ErrCode.COMMON_ERROR, "write content failed: " + e.getMessage());
} catch (UserException ue) {
- LOG.error("connect to s3 failed: ", ue);
+ LOG.warn("connect to s3 failed: ", ue);
return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage());
+ } catch (Exception e) {
+ LOG.warn("connect to s3 failed: ", e);
+ return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + e.getMessage());
}
}
@@ -323,13 +326,13 @@ public class S3Storage extends BlobStorage {
LOG.info("delete file " + remotePath + " success: " + response.toString());
return Status.OK;
} catch (S3Exception e) {
- LOG.error("delete file failed: ", e);
+ LOG.warn("delete file failed: ", e);
if (e.statusCode() == HttpStatus.SC_NOT_FOUND) {
return Status.OK;
}
return new Status(Status.ErrCode.COMMON_ERROR, "delete file failed: " + e.getMessage());
} catch (UserException ue) {
- LOG.error("connect to s3 failed: ", ue);
+ LOG.warn("connect to s3 failed: ", ue);
return new Status(Status.ErrCode.COMMON_ERROR, "connect to s3 failed: " + ue.getMessage());
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index b411c2f46c..6b251281be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -17,6 +17,8 @@
package org.apache.doris.catalog;
+import org.apache.doris.backup.S3Storage;
+import org.apache.doris.backup.Status;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.proc.BaseProcResult;
@@ -29,6 +31,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -66,6 +69,8 @@ public class S3Resource extends Resource {
public static final String S3_ROOT_PATH = "AWS_ROOT_PATH";
public static final String S3_BUCKET = "AWS_BUCKET";
+ private static final String S3_VALIDITY_CHECK = "s3_validity_check";
+
// optional
public static final String S3_TOKEN = "AWS_TOKEN";
public static final String USE_PATH_STYLE = "use_path_style";
@@ -102,12 +107,55 @@ public class S3Resource extends Resource {
checkRequiredProperty(S3_REGION);
checkRequiredProperty(S3_ACCESS_KEY);
checkRequiredProperty(S3_SECRET_KEY);
+ checkRequiredProperty(S3_BUCKET);
+
+ // default need check resource conf valid, so need fix ut and regression case
+ boolean needCheck = !properties.containsKey(S3_VALIDITY_CHECK)
+ || Boolean.parseBoolean(properties.get(S3_VALIDITY_CHECK));
+ LOG.debug("s3 info need check validity : {}", needCheck);
+ if (needCheck) {
+ boolean available = pingS3();
+ if (!available) {
+ throw new DdlException("S3 can't use, please check your properties");
+ }
+ }
+
// optional
checkOptionalProperty(S3_MAX_CONNECTIONS, DEFAULT_S3_MAX_CONNECTIONS);
checkOptionalProperty(S3_REQUEST_TIMEOUT_MS, DEFAULT_S3_REQUEST_TIMEOUT_MS);
checkOptionalProperty(S3_CONNECTION_TIMEOUT_MS, DEFAULT_S3_CONNECTION_TIMEOUT_MS);
}
+ private boolean pingS3() {
+ String bucket = "s3://" + properties.getOrDefault(S3_BUCKET, "") + "/";
+ Map<String, String> propertiesPing = new HashMap<>();
+ propertiesPing.put("AWS_ACCESS_KEY", properties.getOrDefault(S3_ACCESS_KEY, ""));
+ propertiesPing.put("AWS_SECRET_KEY", properties.getOrDefault(S3_SECRET_KEY, ""));
+ propertiesPing.put("AWS_ENDPOINT", "http://" + properties.getOrDefault(S3_ENDPOINT, ""));
+ propertiesPing.put("AWS_REGION", properties.getOrDefault(S3_REGION, ""));
+ propertiesPing.put(S3Resource.USE_PATH_STYLE, "false");
+ S3Storage storage = new S3Storage(propertiesPing);
+
+ String testFile = bucket + properties.getOrDefault(S3_ROOT_PATH, "") + "/test-object-valid.txt";
+ String content = "doris will be better";
+ try {
+ Status status = storage.directUpload(content, testFile);
+ if (status != Status.OK) {
+ LOG.warn("ping update file status: {}, properties: {}", status, propertiesPing);
+ return false;
+ }
+ } finally {
+ Status delete = storage.delete(testFile);
+ if (delete != Status.OK) {
+ LOG.warn("ping delete file status: {}, properties: {}", delete, propertiesPing);
+ return false;
+ }
+ }
+
+ LOG.info("success to ping s3");
+ return true;
+ }
+
private void checkRequiredProperty(String propertyKey) throws DdlException {
String value = properties.get(propertyKey);
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
index cdcc767e9a..8616df9429 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
@@ -140,7 +140,7 @@ public class AlterTest {
+ " \"AWS_ROOT_PATH\" = \"/path/to/root\",\n" + " \"AWS_ACCESS_KEY\" = \"bbb\",\n"
+ " \"AWS_SECRET_KEY\" = \"aaaa\",\n" + " \"AWS_MAX_CONNECTIONS\" = \"50\",\n"
+ " \"AWS_REQUEST_TIMEOUT_MS\" = \"3000\",\n" + " \"AWS_CONNECTION_TIMEOUT_MS\" = \"1000\",\n"
- + " \"AWS_BUCKET\" = \"test-bucket\"\n"
+ + " \"AWS_BUCKET\" = \"test-bucket\", \"s3_validity_check\" = \"false\"\n"
+ ");");
createRemoteStorageResource(
@@ -149,7 +149,7 @@ public class AlterTest {
+ " \"AWS_ROOT_PATH\" = \"/path/to/root\",\n" + " \"AWS_ACCESS_KEY\" = \"bbb\",\n"
+ " \"AWS_SECRET_KEY\" = \"aaaa\",\n" + " \"AWS_MAX_CONNECTIONS\" = \"50\",\n"
+ " \"AWS_REQUEST_TIMEOUT_MS\" = \"3000\",\n" + " \"AWS_CONNECTION_TIMEOUT_MS\" = \"1000\",\n"
- + " \"AWS_BUCKET\" = \"test-bucket\"\n"
+ + " \"AWS_BUCKET\" = \"test-bucket\", \"s3_validity_check\" = \"false\"\n"
+ ");");
createRemoteStoragePolicy(
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ResourceMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ResourceMgrTest.java
index ab1b3679e6..8da7f219ce 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ResourceMgrTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ResourceMgrTest.java
@@ -94,6 +94,7 @@ public class ResourceMgrTest {
s3Properties.put("AWS_ACCESS_KEY", s3AccessKey);
s3Properties.put("AWS_SECRET_KEY", s3SecretKey);
s3Properties.put("AWS_BUCKET", "test-bucket");
+ s3Properties.put("s3_validity_check", "false");
analyzer = AccessTestUtil.fetchAdminAnalyzer(true);
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
index 217448e5c4..6ef82241b2 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
@@ -81,6 +81,7 @@ public class S3ResourceTest {
s3Properties.put("AWS_ACCESS_KEY", s3AccessKey);
s3Properties.put("AWS_SECRET_KEY", s3SecretKey);
s3Properties.put("AWS_BUCKET", s3Bucket);
+ s3Properties.put("s3_validity_check", "false");
analyzer = AccessTestUtil.fetchAdminAnalyzer(true);
}
@@ -115,6 +116,7 @@ public class S3ResourceTest {
s3Properties.put("AWS_MAX_CONNECTIONS", "100");
s3Properties.put("AWS_REQUEST_TIMEOUT_MS", "2000");
s3Properties.put("AWS_CONNECTION_TIMEOUT_MS", "2000");
+ s3Properties.put("s3_validity_check", "false");
stmt = new CreateResourceStmt(true, false, name, s3Properties);
stmt.analyze(analyzer);
@@ -167,6 +169,7 @@ public class S3ResourceTest {
properties.put("AWS_ACCESS_KEY", "xxx");
properties.put("AWS_SECRET_KEY", "yyy");
properties.put("AWS_BUCKET", "test-bucket");
+ properties.put("s3_validity_check", "false");
S3Resource s3Resource2 = new S3Resource("s3_2");
s3Resource2.setProperties(properties);
s3Resource2.write(s3Dos);
diff --git a/regression-test/suites/cold_heat_separation/policy/alter.groovy b/regression-test/suites/cold_heat_separation/policy/alter.groovy
index 07166df0d3..bf58d24b35 100644
--- a/regression-test/suites/cold_heat_separation/policy/alter.groovy
+++ b/regression-test/suites/cold_heat_separation/policy/alter.groovy
@@ -26,7 +26,7 @@ suite("alter_policy") {
CREATE RESOURCE "has_resouce_policy_alter"
PROPERTIES(
"type"="s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_REGION" = "bj",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
@@ -34,7 +34,8 @@ suite("alter_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
@@ -95,7 +96,7 @@ suite("alter_policy") {
// [[has_resouce_policy_alter, s3, AWS_ACCESS_KEY, 6666],
// [has_resouce_policy_alter, s3, AWS_BUCKET, test-bucket],
// [has_resouce_policy_alter, s3, AWS_CONNECTION_TIMEOUT_MS, 2222],
- // [has_resouce_policy_alter, s3, AWS_ENDPOINT, http://bj.s3.comaaaa],
+ // [has_resouce_policy_alter, s3, AWS_ENDPOINT, bj.s3.comaaaa],
// [has_resouce_policy_alter, s3, AWS_MAX_CONNECTIONS, 1111],
// [has_resouce_policy_alter, s3, AWS_REGION, bj],
// [has_resouce_policy_alter, s3, AWS_REQUEST_TIMEOUT_MS, 7777],
@@ -109,7 +110,7 @@ suite("alter_policy") {
// AWS_CONNECTION_TIMEOUT_MS
assertEquals(show_alter_result[2][3], "2222")
// AWS_ENDPOINT
- assertEquals(show_alter_result[3][3], "http://bj.s3.comaaaa")
+ assertEquals(show_alter_result[3][3], "bj.s3.comaaaa")
// AWS_MAX_CONNECTIONS
assertEquals(show_alter_result[4][3], "1111")
// AWS_REGION
@@ -121,7 +122,7 @@ suite("alter_policy") {
// AWS_SECRET_KEY
assertEquals(show_alter_result[8][3], "******")
// type
- assertEquals(show_alter_result[9][3], "s3")
+ assertEquals(show_alter_result[10][3], "s3")
def storage_exist = { name ->
def show_storage_policy = sql """
diff --git a/regression-test/suites/cold_heat_separation/policy/create.groovy b/regression-test/suites/cold_heat_separation/policy/create.groovy
index c217486ad1..21b217d7b5 100644
--- a/regression-test/suites/cold_heat_separation/policy/create.groovy
+++ b/regression-test/suites/cold_heat_separation/policy/create.groovy
@@ -29,7 +29,7 @@ suite("create_policy") {
CREATE RESOURCE "crete_policy_1"
PROPERTIES(
"type" = "s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_REGION" = "bj",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
@@ -37,20 +37,21 @@ suite("create_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_sucess = sql """
SHOW RESOURCES WHERE NAME = "crete_policy_1";
"""
- assertEquals(create_sucess.size(), 10)
+ assertEquals(create_sucess.size(), 11)
def failed_cannot_create_duplicate_name_resources = try_sql """
CREATE RESOURCE "crete_policy_1"
PROPERTIES(
"type" = "s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaab",
+ "AWS_ENDPOINT" = "bj.s3.comaaaab",
"AWS_REGION" = "bjc",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
@@ -58,7 +59,8 @@ suite("create_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
@@ -74,7 +76,7 @@ suite("create_policy") {
def failed_create_1 = try_sql """
CREATE RESOURCE "crete_policy_2"
PROPERTIES(
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_REGION" = "bj",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
@@ -82,7 +84,8 @@ suite("create_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
// errCode = 2, detailMessage = Resource type can't be null
@@ -101,7 +104,8 @@ suite("create_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
// errCode = 2, detailMessage = Missing [AWS_ENDPOINT] in properties.
@@ -113,14 +117,15 @@ suite("create_policy") {
CREATE RESOURCE "crete_policy_2"
PROPERTIES(
"type" = "s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
"AWS_SECRET_KEY" = "aaaa",
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
// errCode = 2, detailMessage = Missing [AWS_REGION] in properties.
@@ -132,14 +137,15 @@ suite("create_policy") {
CREATE RESOURCE "crete_policy_2"
PROPERTIES(
"type"="s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_REGION" = "bj",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
// errCode = 2, detailMessage = Missing [AWS_ACCESS_KEY] in properties.
@@ -151,14 +157,15 @@ suite("create_policy") {
CREATE RESOURCE "crete_policy_2"
PROPERTIES(
"type"="s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_REGION" = "bj",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
// errCode = 2, detailMessage = Missing [AWS_SECRET_KEY] in properties.
@@ -175,11 +182,12 @@ suite("create_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
// "AWS_MAX_CONNECTIONS" = "50", "AWS_REQUEST_TIMEOUT_MS" = "3000","AWS_CONNECTION_TIMEOUT_MS" = "1000"
@@ -207,11 +215,12 @@ suite("create_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
@@ -230,11 +239,12 @@ suite("create_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
@@ -253,11 +263,12 @@ suite("create_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
@@ -278,11 +289,12 @@ suite("create_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
@@ -306,7 +318,8 @@ suite("create_policy") {
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
diff --git a/regression-test/suites/cold_heat_separation/policy/drop.groovy b/regression-test/suites/cold_heat_separation/policy/drop.groovy
index e947e20c7b..c04b1afdcb 100644
--- a/regression-test/suites/cold_heat_separation/policy/drop.groovy
+++ b/regression-test/suites/cold_heat_separation/policy/drop.groovy
@@ -39,7 +39,7 @@ suite("drop_policy") {
CREATE RESOURCE "resouce_policy_drop"
PROPERTIES(
"type"="s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_REGION" = "bj",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
@@ -47,7 +47,8 @@ suite("drop_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
}
@@ -62,7 +63,7 @@ suite("drop_policy") {
CREATE RESOURCE "resouce_policy_drop"
PROPERTIES(
"type"="s3",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_REGION" = "bj",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_ACCESS_KEY" = "bbba",
@@ -70,7 +71,8 @@ suite("drop_policy") {
"AWS_MAX_CONNECTIONS" = "50",
"AWS_REQUEST_TIMEOUT_MS" = "3000",
"AWS_CONNECTION_TIMEOUT_MS" = "1000",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
diff --git a/regression-test/suites/cold_heat_separation/policy/show.groovy b/regression-test/suites/cold_heat_separation/policy/show.groovy
index 06a4340372..3a62e50e43 100644
--- a/regression-test/suites/cold_heat_separation/policy/show.groovy
+++ b/regression-test/suites/cold_heat_separation/policy/show.groovy
@@ -38,11 +38,12 @@ suite("show_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
diff --git a/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy
index bff146f6e0..0c02dc2888 100644
--- a/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy
+++ b/regression-test/suites/cold_heat_separation/use_policy/alter_table_add_policy.groovy
@@ -57,11 +57,12 @@ suite("add_table_policy_by_alter_table") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
@@ -92,11 +93,12 @@ suite("add_table_policy_by_alter_table") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
diff --git a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy
index 3a1b4ff833..d60fc1e0d7 100644
--- a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy
+++ b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_partition_policy.groovy
@@ -58,11 +58,12 @@ suite("create_table_use_partition_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
@@ -82,11 +83,12 @@ suite("create_table_use_partition_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
diff --git a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy
index 871d07c9e9..bb70c834ed 100644
--- a/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy
+++ b/regression-test/suites/cold_heat_separation/use_policy/create_table_use_policy.groovy
@@ -60,11 +60,12 @@ suite("create_table_use_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
diff --git a/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy
index c7dc83379e..0998cc4b1d 100644
--- a/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy
+++ b/regression-test/suites/cold_heat_separation/use_policy/modify_partition_add_policy.groovy
@@ -83,11 +83,12 @@ suite("add_table_policy_by_modify_partition") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
@@ -106,12 +107,6 @@ suite("add_table_policy_by_modify_partition") {
// OK
assertEquals(alter_table_partition_try_again_result.size(), 1);
- def alter_table_when_table_partition_has_storage_policy_result = try_sql """
- ALTER TABLE create_table_partition MODIFY PARTITION p1992 SET("storage_policy"="created_create_table_partition_alter_policy");
- """
- // errCode = 2, detailMessage = Do not support alter table's partition storage policy , this table [create_table_partition] and partition [p1992] has storage policy created_create_table_partition_alter_policy
- assertEquals(alter_table_when_table_partition_has_storage_policy_result, null);
-
sql """
DROP TABLE IF EXISTS create_table_partition;
"""
diff --git a/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy b/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy
index fb65808514..45bd4a77d3 100644
--- a/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy
+++ b/regression-test/suites/cold_heat_separation/use_policy/use_default_storage_policy.groovy
@@ -48,11 +48,12 @@ suite("use_default_storage_policy") {
PROPERTIES(
"type"="s3",
"AWS_REGION" = "bj",
- "AWS_ENDPOINT" = "http://bj.s3.comaaaa",
+ "AWS_ENDPOINT" = "bj.s3.comaaaa",
"AWS_ROOT_PATH" = "path/to/rootaaaa",
"AWS_SECRET_KEY" = "aaaa",
"AWS_ACCESS_KEY" = "bbba",
- "AWS_BUCKET" = "test-bucket"
+ "AWS_BUCKET" = "test-bucket",
+ "s3_validity_check" = "false"
);
"""
def create_succ_1 = try_sql """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org