You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/18 12:22:07 UTC
[doris] branch master updated: [fix](multi-catalog)fix s3 check, complete catalog properties (#20591)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8af77315e0 [fix](multi-catalog)fix s3 check, complete catalog properties (#20591)
8af77315e0 is described below
commit 8af77315e058fb2b1fa5f1a7f17130cfb8564a81
Author: slothever <18...@users.noreply.github.com>
AuthorDate: Sun Jun 18 20:21:59 2023 +0800
[fix](multi-catalog)fix s3 check, complete catalog properties (#20591)
stability and some fixes
1. fix s3 availability check
2. add independent minio properties
3. add job conf cache
4. remove extra s3 propertie when convert catalog properties
5. add some ut case to check conveted properties
---
.../java/org/apache/doris/catalog/S3Resource.java | 3 +-
.../org/apache/doris/datasource/CatalogMgr.java | 5 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 27 ++---
.../datasource/property/PropertyConverter.java | 27 ++++-
.../datasource/property/S3ClientBEProperties.java | 7 +-
.../property/constants/MinioProperties.java | 39 +++++++
.../property/constants/S3Properties.java | 5 +
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 2 +-
.../org/apache/doris/system/SystemInfoService.java | 2 +-
.../ExternalFileTableValuedFunction.java | 4 +-
.../tablefunction/HdfsTableValuedFunction.java | 28 ++---
.../datasource/property/PropertyConverterTest.java | 115 ++++++++++++++++-----
.../org/apache/doris/persist/DropDbInfoTest.java | 7 +-
13 files changed, 203 insertions(+), 68 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index a12fd15861..b8ef318077 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -127,7 +127,8 @@ public class S3Resource extends Resource {
propertiesPing.put(S3Properties.Env.SECRET_KEY, credential.getSecretKey());
propertiesPing.put(S3Properties.Env.ENDPOINT, credential.getEndpoint());
propertiesPing.put(S3Properties.Env.REGION, credential.getRegion());
- propertiesPing.put(PropertyConverter.USE_PATH_STYLE, "false");
+ propertiesPing.put(PropertyConverter.USE_PATH_STYLE,
+ properties.getOrDefault(PropertyConverter.USE_PATH_STYLE, "false"));
properties.putAll(propertiesPing);
S3FileSystem fileSystem = new S3FileSystem(properties);
String testFile = bucket + rootPath + "/test-object-valid.txt";
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 11a10c2a78..67b9bb6c8d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -63,6 +63,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -417,7 +418,9 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
if (!Strings.isNullOrEmpty(catalog.getResource())) {
rows.add(Arrays.asList("resource", catalog.getResource()));
}
- for (Map.Entry<String, String> elem : catalog.getProperties().entrySet()) {
+ // use tree map to maintain display order, making it easier to view properties
+ Map<String, String> sortedMap = new TreeMap<>(catalog.getProperties()).descendingMap();
+ for (Map.Entry<String, String> elem : sortedMap.entrySet()) {
if (PrintableMap.HIDDEN_KEY.contains(elem.getKey())) {
continue;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index cbc8631883..5777cf49a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -108,6 +108,7 @@ public class HiveMetaStoreCache {
private static final String HIVE_TRANSACTIONAL_ORC_BUCKET_PREFIX = "bucket_";
private HMSExternalCatalog catalog;
+ private JobConf jobConf;
private Executor executor;
@@ -153,6 +154,8 @@ public class HiveMetaStoreCache {
* generate a filecache and set to fileCacheRef
*/
public void setNewFileCache() {
+ // init or refresh job conf
+ setJobConf();
// if the file.meta.cache.ttl-second is equal or greater than 0, the cache expired will be set to that value
int fileMetaCacheTtlSecond = NumberUtils.toInt(
(catalog.getProperties().get(HMSExternalCatalog.FILE_META_CACHE_TTL_SECOND)),
@@ -325,16 +328,6 @@ public class HiveMetaStoreCache {
try {
Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader());
String finalLocation = S3Util.convertToS3IfNecessary(key.location);
- JobConf jobConf = getJobConf();
- // For Tez engine, it may generate subdirectories for "union" query.
- // So there may be files and directories in the table directory at the same time. eg:
- // /us£er/hive/warehouse/region_tmp_union_all2/000000_0
- // /user/hive/warehouse/region_tmp_union_all2/1
- // /user/hive/warehouse/region_tmp_union_all2/2
- // So we need to set this config to support visit dir recursively.
- // Otherwise, getSplits() may throw exception: "Not a file xxx"
- // https://blog.actorsfit.com/a?ID=00550-ce56ec63-1bff-4b0c-a6f7-447b93efaa31
- jobConf.set("mapreduce.input.fileinputformat.input.dir.recursive", "true");
FileInputFormat.setInputPaths(jobConf, finalLocation);
try {
FileCacheValue result;
@@ -373,12 +366,21 @@ public class HiveMetaStoreCache {
}
}
- private JobConf getJobConf() {
+ private synchronized void setJobConf() {
Configuration configuration = new HdfsConfiguration();
for (Map.Entry<String, String> entry : catalog.getCatalogProperty().getHadoopProperties().entrySet()) {
configuration.set(entry.getKey(), entry.getValue());
}
- return new JobConf(configuration);
+ jobConf = new JobConf(configuration);
+ // For Tez engine, it may generate subdirectories for "union" query.
+ // So there may be files and directories in the table directory at the same time. eg:
+ // /us£er/hive/warehouse/region_tmp_union_all2/000000_0
+ // /user/hive/warehouse/region_tmp_union_all2/1
+ // /user/hive/warehouse/region_tmp_union_all2/2
+ // So we need to set this config to support visit dir recursively.
+ // Otherwise, getSplits() may throw exception: "Not a file xxx"
+ // https://blog.actorsfit.com/a?ID=00550-ce56ec63-1bff-4b0c-a6f7-447b93efaa31
+ jobConf.set("mapreduce.input.fileinputformat.input.dir.recursive", "true");
}
public HivePartitionValues getPartitionValues(String dbName, String tblName, List<Type> types) {
@@ -671,7 +673,6 @@ public class HiveMetaStoreCache {
public List<FileCacheValue> getFilesByTransaction(List<HivePartition> partitions, ValidWriteIdList validWriteIds,
boolean isFullAcid) {
List<FileCacheValue> fileCacheValues = Lists.newArrayList();
- JobConf jobConf = getJobConf();
String remoteUser = jobConf.get(HdfsResource.HADOOP_USER_NAME);
try {
for (HivePartition partition : partitions) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index 61a685abfa..6f8ff8fe44 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -26,6 +26,7 @@ import org.apache.doris.datasource.property.constants.DLFProperties;
import org.apache.doris.datasource.property.constants.GCSProperties;
import org.apache.doris.datasource.property.constants.GlueProperties;
import org.apache.doris.datasource.property.constants.HMSProperties;
+import org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.datasource.property.constants.S3Properties;
@@ -80,15 +81,25 @@ public class PropertyConverter {
|| props.containsKey(DataLakeConfig.CATALOG_ENDPOINT)) {
metaProperties = convertToDLFProperties(props, DLFProperties.getCredential(props));
} else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
- // checkout env in the end
- // if meet AWS_XXX properties, convert to s3 properties
- return convertToS3EnvProperties(props, S3Properties.getEnvironmentCredentialWithEndpoint(props), true);
+ if (!hasS3Properties(props)) {
+ // checkout env in the end
+ // if meet AWS_XXX properties, convert to s3 properties
+ return convertToS3EnvProperties(props, S3Properties.getEnvironmentCredentialWithEndpoint(props), true);
+ }
}
metaProperties.putAll(props);
metaProperties.putAll(S3ClientBEProperties.getBeFSProperties(props));
return metaProperties;
}
+ private static boolean hasS3Properties(Map<String, String> props) {
+ return props.containsKey(ObsProperties.ENDPOINT)
+ || props.containsKey(GCSProperties.ENDPOINT)
+ || props.containsKey(OssProperties.ENDPOINT)
+ || props.containsKey(CosProperties.ENDPOINT)
+ || props.containsKey(MinioProperties.ENDPOINT);
+ }
+
/**
* Convert properties defined at doris to FE S3 client properties
* Support other cloud client here.
@@ -102,6 +113,8 @@ public class PropertyConverter {
return convertToOSSProperties(props, OssProperties.getCredential(props));
} else if (props.containsKey(CosProperties.ENDPOINT)) {
return convertToCOSProperties(props, CosProperties.getCredential(props));
+ } else if (props.containsKey(MinioProperties.ENDPOINT)) {
+ return convertToMinioProperties(props, MinioProperties.getCredential(props));
} else if (props.containsKey(S3Properties.ENDPOINT)) {
return convertToS3Properties(props, S3Properties.getCredential(props));
} else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
@@ -179,6 +192,8 @@ public class PropertyConverter {
}
setS3FsAccess(s3Properties, properties, credential);
s3Properties.putAll(properties);
+ // remove extra meta properties
+ S3Properties.FS_KEYS.forEach(s3Properties::remove);
return s3Properties;
}
@@ -235,6 +250,12 @@ public class PropertyConverter {
return convertToS3Properties(S3Properties.prefixToS3(props), credential);
}
+ private static Map<String, String> convertToMinioProperties(Map<String, String> props, CloudCredential credential) {
+ // minio does not have region, use an arbitrary one.
+ props.put(MinioProperties.REGION, "us-east-1");
+ return convertToS3Properties(S3Properties.prefixToS3(props), credential);
+ }
+
private static Map<String, String> convertToDLFProperties(Map<String, String> props, CloudCredential credential) {
getPropertiesFromDLFConf(props);
// if configure DLF properties in catalog properties, use them to override config in hive-site.xml
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
index 90efd12de1..b0639be2df 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/S3ClientBEProperties.java
@@ -19,6 +19,7 @@ package org.apache.doris.datasource.property;
import org.apache.doris.datasource.property.constants.CosProperties;
import org.apache.doris.datasource.property.constants.GCSProperties;
+import org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.datasource.property.constants.S3Properties;
@@ -33,7 +34,11 @@ public class S3ClientBEProperties {
* On BE, should use properties like AWS_XXX.
*/
public static Map<String, String> getBeFSProperties(Map<String, String> properties) {
- if (properties.containsKey(S3Properties.ENDPOINT)) {
+ if (properties.containsKey(MinioProperties.ENDPOINT)) {
+ // minio does not have region, use an arbitrary one.
+ properties.put(MinioProperties.REGION, "us-east-1");
+ return getBeAWSPropertiesFromS3(S3Properties.prefixToS3(properties));
+ } else if (properties.containsKey(S3Properties.ENDPOINT)) {
// s3,oss,cos,obs use this.
return getBeAWSPropertiesFromS3(properties);
} else if (properties.containsKey(ObsProperties.ENDPOINT)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java
new file mode 100644
index 0000000000..c444384220
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.constants;
+
+import org.apache.doris.datasource.credentials.CloudCredential;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+public class MinioProperties extends BaseProperties {
+
+ public static final String MINIO_PREFIX = "minio.";
+ public static final String ENDPOINT = "minio.endpoint";
+ public static final String REGION = "minio.region";
+ public static final String ACCESS_KEY = "minio.access_key";
+ public static final String SECRET_KEY = "minio.secret_key";
+ public static final String SESSION_TOKEN = "minio.session_token";
+ public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY, REGION);
+
+ public static CloudCredential getCredential(Map<String, String> props) {
+ return getCloudCredential(props, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
index e42eba9b80..4b9d7a1023 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java
@@ -57,6 +57,8 @@ public class S3Properties extends BaseProperties {
public static final String VALIDITY_CHECK = "s3_validity_check";
public static final List<String> REQUIRED_FIELDS = Arrays.asList(ENDPOINT, ACCESS_KEY, SECRET_KEY);
public static final List<String> TVF_REQUIRED_FIELDS = Arrays.asList(ACCESS_KEY, SECRET_KEY);
+ public static final List<String> FS_KEYS = Arrays.asList(ENDPOINT, REGION, ACCESS_KEY, SECRET_KEY, SESSION_TOKEN,
+ ROOT_PATH, BUCKET, MAX_CONNECTIONS, REQUEST_TIMEOUT_MS, CONNECTION_TIMEOUT_MS);
public static final List<String> AWS_CREDENTIALS_PROVIDERS = Arrays.asList(
DataLakeAWSCredentialsProvider.class.getName(),
@@ -140,6 +142,9 @@ public class S3Properties extends BaseProperties {
} else if (entry.getKey().startsWith(ObsProperties.OBS_PREFIX)) {
String s3Key = entry.getKey().replace(ObsProperties.OBS_PREFIX, S3Properties.S3_PREFIX);
s3Properties.put(s3Key, entry.getValue());
+ } else if (entry.getKey().startsWith(MinioProperties.MINIO_PREFIX)) {
+ String s3Key = entry.getKey().replace(MinioProperties.MINIO_PREFIX, S3Properties.S3_PREFIX);
+ s3Properties.put(s3Key, entry.getValue());
}
}
return s3Properties;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
index 677ca658be..9f72595ad8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
@@ -126,7 +126,7 @@ public class DFSFileSystem extends RemoteFileSystem {
String hadoopUserName = conf.get(HdfsResource.HADOOP_USER_NAME);
if (hadoopUserName == null) {
hadoopUserName = "hadoop";
- LOG.warn("hadoop.username is unset, use default user: hadoop");
+ LOG.debug(HdfsResource.HADOOP_USER_NAME + " is unset, use default user: hadoop");
}
return UserGroupInformation.createRemoteUser(hadoopUserName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
index da81d4202b..f93f906cc3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java
@@ -812,7 +812,7 @@ public class SystemInfoService {
public void checkAvailableCapacity() throws DdlException {
if (getAvailableCapacityB() <= 0L) {
- throw new DdlException("System has no available disk capacity");
+ throw new DdlException("System has no available disk capacity or no available BE nodes");
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 5d3aef2213..994353af37 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -166,7 +166,7 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
public String getFsName() {
TFileType fileType = getTFileType();
if (fileType == TFileType.FILE_HDFS) {
- return locationProperties.get(HdfsTableValuedFunction.HADOOP_FS_NAME);
+ return locationProperties.get(HdfsResource.HADOOP_FS_NAME);
} else if (fileType == TFileType.FILE_S3) {
return locationProperties.get(S3Properties.ENDPOINT);
}
@@ -445,7 +445,7 @@ public abstract class ExternalFileTableValuedFunction extends TableValuedFunctio
fileScanRangeParams.setFileAttributes(getFileAttributes());
if (getTFileType() == TFileType.FILE_HDFS) {
THdfsParams tHdfsParams = HdfsResource.generateHdfsParam(locationProperties);
- String fsNmae = getLocationProperties().get(HdfsTableValuedFunction.HADOOP_FS_NAME);
+ String fsNmae = getLocationProperties().get(HdfsResource.HADOOP_FS_NAME);
tHdfsParams.setFsName(fsNmae);
fileScanRangeParams.setHdfsParams(tHdfsParams);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
index 9c845103f9..ea37ec410f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java
@@ -20,6 +20,7 @@ package org.apache.doris.tablefunction;
import org.apache.doris.analysis.BrokerDesc;
import org.apache.doris.analysis.ExportStmt;
import org.apache.doris.analysis.StorageBackend.StorageType;
+import org.apache.doris.catalog.HdfsResource;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.util.URI;
import org.apache.doris.thrift.TFileType;
@@ -41,24 +42,17 @@ public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction {
public static final String NAME = "hdfs";
public static final String HDFS_URI = "uri";
- public static String HADOOP_FS_NAME = "fs.defaultFS";
// simple or kerberos
- public static String HADOOP_SECURITY_AUTHENTICATION = "hadoop.security.authentication";
- public static String HADOOP_USER_NAME = "hadoop.username";
- public static String HADOOP_KERBEROS_PRINCIPAL = "hadoop.kerberos.principal";
- public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab";
- public static String HADOOP_SHORT_CIRCUIT = "dfs.client.read.shortcircuit";
- public static String HADOOP_SOCKET_PATH = "dfs.domain.socket.path";
private static final ImmutableSet<String> LOCATION_PROPERTIES = new ImmutableSet.Builder<String>()
.add(HDFS_URI)
- .add(HADOOP_SECURITY_AUTHENTICATION)
- .add(HADOOP_FS_NAME)
- .add(HADOOP_USER_NAME)
- .add(HADOOP_KERBEROS_PRINCIPAL)
- .add(HADOOP_KERBEROS_KEYTAB)
- .add(HADOOP_SHORT_CIRCUIT)
- .add(HADOOP_SOCKET_PATH)
+ .add(HdfsResource.HADOOP_SECURITY_AUTHENTICATION)
+ .add(HdfsResource.HADOOP_FS_NAME)
+ .add(HdfsResource.HADOOP_USER_NAME)
+ .add(HdfsResource.HADOOP_KERBEROS_PRINCIPAL)
+ .add(HdfsResource.HADOOP_KERBEROS_KEYTAB)
+ .add(HdfsResource.HADOOP_SHORT_CIRCUIT)
+ .add(HdfsResource.HADOOP_SOCKET_PATH)
.build();
private URI hdfsUri;
@@ -72,8 +66,8 @@ public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction {
fileFormatParams.put(key, params.get(key));
} else {
// because HADOOP_FS_NAME contains upper and lower case
- if (HADOOP_FS_NAME.equalsIgnoreCase(key)) {
- locationProperties.put(HADOOP_FS_NAME, params.get(key));
+ if (HdfsResource.HADOOP_FS_NAME.equalsIgnoreCase(key)) {
+ locationProperties.put(HdfsResource.HADOOP_FS_NAME, params.get(key));
} else {
locationProperties.put(key, params.get(key));
}
@@ -85,7 +79,7 @@ public class HdfsTableValuedFunction extends ExternalFileTableValuedFunction {
}
ExportStmt.checkPath(locationProperties.get(HDFS_URI), StorageType.HDFS);
hdfsUri = URI.create(locationProperties.get(HDFS_URI));
- filePath = locationProperties.get(HADOOP_FS_NAME) + hdfsUri.getPath();
+ filePath = locationProperties.get(HdfsResource.HADOOP_FS_NAME) + hdfsUri.getPath();
parseProperties(fileFormatParams);
parseFile();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index 7862347c19..819385b847 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.property;
import org.apache.doris.analysis.CreateCatalogStmt;
import org.apache.doris.analysis.CreateRepositoryStmt;
import org.apache.doris.analysis.CreateResourceStmt;
+import org.apache.doris.analysis.DropCatalogStmt;
import org.apache.doris.analysis.OutFileClause;
import org.apache.doris.analysis.QueryStmt;
import org.apache.doris.analysis.SelectStmt;
@@ -29,10 +30,17 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Resource;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.Pair;
import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.datasource.HMSExternalCatalog;
+import org.apache.doris.datasource.property.constants.CosProperties;
+import org.apache.doris.datasource.property.constants.MinioProperties;
+import org.apache.doris.datasource.property.constants.ObsProperties;
+import org.apache.doris.datasource.property.constants.OssProperties;
+import org.apache.doris.datasource.property.constants.S3Properties;
+import org.apache.doris.meta.MetaContext;
import org.apache.doris.tablefunction.S3TableValuedFunction;
import org.apache.doris.thrift.TFileFormatType;
import org.apache.doris.utframe.TestWithFeService;
@@ -41,11 +49,18 @@ import com.google.common.collect.ImmutableList;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
public class PropertyConverterTest extends TestWithFeService {
+ private final Set<String> checkSet = new HashSet<>();
+ private final Map<String, String> expectedCredential = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
+
@Override
protected void runBeforeAll() throws Exception {
createDorisCluster();
@@ -53,6 +68,12 @@ public class PropertyConverterTest extends TestWithFeService {
useDatabase("mock_db");
createTable("create table mock_tbl1 \n" + "(k1 int, k2 int) distributed by hash(k1) buckets 1\n"
+ "properties('replication_num' = '1');");
+
+ List<String> withoutPrefix = ImmutableList.of("endpoint", "access_key", "secret_key");
+ checkSet.addAll(withoutPrefix);
+ checkSet.addAll(S3Properties.Env.REQUIRED_FIELDS);
+ expectedCredential.put("access_key", "akk");
+ expectedCredential.put("secret_key", "skk");
}
@Test
@@ -130,6 +151,7 @@ public class PropertyConverterTest extends TestWithFeService {
Resource newResource = Resource.fromStmt(analyzedResourceStmtNew);
// will add converted properties
Assertions.assertEquals(newResource.getCopiedProperties().size(), 14);
+
}
@Test
@@ -272,8 +294,9 @@ public class PropertyConverterTest extends TestWithFeService {
+ " 'aws.glue.secret-key' = 'skk',\n"
+ " 'aws.region' = 'us-east-1'\n"
+ ");";
+ String catalogName = "hms_glue_old";
CreateCatalogStmt analyzedStmt = createStmt(queryOld);
- HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_glue_old");
+ HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, catalogName);
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 20);
@@ -288,8 +311,9 @@ public class PropertyConverterTest extends TestWithFeService {
+ " 'glue.access_key' = 'akk',\n"
+ " 'glue.secret_key' = 'skk'\n"
+ ");";
+ catalogName = "hms_glue";
CreateCatalogStmt analyzedStmtNew = createStmt(query);
- HMSExternalCatalog catalogNew = createAndGetCatalog(analyzedStmtNew, "hms_glue");
+ HMSExternalCatalog catalogNew = createAndGetCatalog(analyzedStmtNew, catalogName);
Map<String, String> propertiesNew = catalogNew.getCatalogProperty().getProperties();
Assertions.assertEquals(propertiesNew.size(), 20);
@@ -313,43 +337,78 @@ public class PropertyConverterTest extends TestWithFeService {
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
Assertions.assertEquals(hdProps.size(), 16);
+
+ Map<String, String> expectedMetaProperties = new HashMap<>();
+ expectedMetaProperties.put("endpoint", "obs.cn-north-4.myhuaweicloud.com");
+ expectedMetaProperties.put("AWS_ENDPOINT", "obs.cn-north-4.myhuaweicloud.com");
+ expectedMetaProperties.putAll(expectedCredential);
+ checkExpectedProperties(ObsProperties.OBS_PREFIX, properties, expectedMetaProperties);
}
@Test
- public void testCOSCatalogPropertiesConverter() throws Exception {
- String query = "create catalog hms_cos properties (\n"
+ public void testS3CompatibleCatalogPropertiesConverter() throws Exception {
+ String catalogName0 = "hms_cos";
+ String query0 = "create catalog " + catalogName0 + " properties (\n"
+ " 'type'='hms',\n"
+ " 'hive.metastore.uris' = 'thrift://172.21.0.1:7004',\n"
+ " 'cos.endpoint' = 'cos.ap-beijing.myqcloud.com',\n"
+ " 'cos.access_key' = 'akk',\n"
- + " 'cos.secret_key' = 'skk',\n"
- + " 'enable.self.splitter'='true'\n"
+ + " 'cos.secret_key' = 'skk'\n"
+ ");";
- CreateCatalogStmt analyzedStmt = createStmt(query);
- HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_cos");
- Map<String, String> properties = catalog.getCatalogProperty().getProperties();
- Assertions.assertEquals(properties.size(), 12);
+ testS3CompatibleCatalogProperties(catalogName0, CosProperties.COS_PREFIX,
+ "cos.ap-beijing.myqcloud.com", query0);
- Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdProps.size(), 24);
+ String catalogName1 = "hms_oss";
+ String query1 = "create catalog " + catalogName1 + " properties (\n"
+ + " 'type'='hms',\n"
+ + " 'hive.metastore.uris' = 'thrift://172.21.0.1:7004',\n"
+ + " 'oss.endpoint' = 'oss.oss-cn-beijing.aliyuncs.com',\n"
+ + " 'oss.access_key' = 'akk',\n"
+ + " 'oss.secret_key' = 'skk'\n"
+ + ");";
+ testS3CompatibleCatalogProperties(catalogName1, OssProperties.OSS_PREFIX,
+ "oss.oss-cn-beijing.aliyuncs.com", query1);
+
+ String catalogName2 = "hms_minio";
+ String query2 = "create catalog " + catalogName2 + " properties (\n"
+ + " 'type'='hms',\n"
+ + " 'hive.metastore.uris' = 'thrift://172.21.0.1:7004',\n"
+ + " 'minio.endpoint' = 'http://127.0.0.1',\n"
+ + " 'minio.access_key' = 'akk',\n"
+ + " 'minio.secret_key' = 'skk'\n"
+ + ");";
+ testS3CompatibleCatalogProperties(catalogName2, MinioProperties.MINIO_PREFIX,
+ "http://127.0.0.1", query2);
}
- @Test
- public void testOSSCatalogPropertiesConverter() throws Exception {
- String query = "create catalog hms_oss properties (\n"
- + " 'type'='hms',\n"
- + " 'hive.metastore.uris' = 'thrift://172.21.0.1:7004',\n"
- + " 'oss.endpoint' = 'oss.oss-cn-beijing.aliyuncs.com',\n"
- + " 'oss.access_key' = 'akk',\n"
- + " 'oss.secret_key' = 'skk'\n"
- + ");";
- CreateCatalogStmt analyzedStmt = createStmt(query);
- HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, "hms_oss");
+ private void testS3CompatibleCatalogProperties(String catalogName, String prefix,
+ String endpoint, String sql) throws Exception {
+ Env.getCurrentEnv().getCatalogMgr().dropCatalog(new DropCatalogStmt(true, catalogName));
+ CreateCatalogStmt analyzedStmt = createStmt(sql);
+ HMSExternalCatalog catalog = createAndGetCatalog(analyzedStmt, catalogName);
Map<String, String> properties = catalog.getCatalogProperty().getProperties();
Assertions.assertEquals(properties.size(), 11);
Map<String, String> hdProps = catalog.getCatalogProperty().getHadoopProperties();
- Assertions.assertEquals(hdProps.size(), 23);
+ Assertions.assertEquals(hdProps.size(), 20);
+
+ Map<String, String> expectedMetaProperties = new HashMap<>();
+ expectedMetaProperties.put("endpoint", endpoint);
+ expectedMetaProperties.put("AWS_ENDPOINT", endpoint);
+ expectedMetaProperties.putAll(expectedCredential);
+ checkExpectedProperties(prefix, properties, expectedMetaProperties);
+ }
+
+ private void checkExpectedProperties(String prefix, Map<String, String> properties,
+ Map<String, String> expectedProperties) {
+ properties.forEach((key, value) -> {
+ if (key.startsWith(prefix)) {
+ String keyToCheck = key.replace(prefix, "");
+ if (checkSet.contains(keyToCheck)) {
+ Assertions.assertEquals(value, expectedProperties.get(keyToCheck));
+ }
+ }
+ });
}
private static HMSExternalCatalog createAndGetCatalog(CreateCatalogStmt analyzedStmt, String name)
@@ -357,4 +416,12 @@ public class PropertyConverterTest extends TestWithFeService {
Env.getCurrentEnv().getCatalogMgr().createCatalog(analyzedStmt);
return (HMSExternalCatalog) Env.getCurrentEnv().getCatalogMgr().getCatalog(name);
}
+
+
+ @Test
+ public void testSerialization() throws Exception {
+ MetaContext metaContext = new MetaContext();
+ metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT);
+ metaContext.setThreadLocalInfo();
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java
index 3548845169..aea8530afe 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java
@@ -26,8 +26,7 @@ import org.junit.Test;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
+import java.nio.file.Files;
public class DropDbInfoTest {
@Test
@@ -39,7 +38,7 @@ public class DropDbInfoTest {
// 1. Write objects to file
File file = new File("./dropDbInfo");
file.createNewFile();
- DataOutputStream dos = new DataOutputStream(new FileOutputStream(file));
+ DataOutputStream dos = new DataOutputStream(Files.newOutputStream(file.toPath()));
DropDbInfo info1 = new DropDbInfo();
info1.write(dos);
@@ -51,7 +50,7 @@ public class DropDbInfoTest {
dos.close();
// 2. Read objects from file
- DataInputStream dis = new DataInputStream(new FileInputStream(file));
+ DataInputStream dis = new DataInputStream(Files.newInputStream(file.toPath()));
DropDbInfo rInfo1 = DropDbInfo.read(dis);
Assert.assertEquals(rInfo1, info1);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org