You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/08/09 02:58:00 UTC
[doris] branch dev-1.1.2 updated (b9188de682 -> d524c813bd)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
from b9188de682 [fix](auditloader)add execute permission for fe_plugins/auditloader/build.sh (#11507)
new 4a1d24b62f [multi-catalog](oss)Support hive external table on Ali oss. (#11489)
new d524c813bd [improvement](orc)improve hdfs scan performance (#11501)
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../doris/catalog/HiveMetaStoreClientHelper.java | 18 +++++++++++++++---
.../java/org/apache/doris/catalog/IcebergTable.java | 3 ++-
.../src/main/java/org/apache/doris/common/Config.java | 6 ++++++
.../main/java/org/apache/doris/common/util/Util.java | 10 ++++++++++
.../java/org/apache/doris/planner/HiveScanNode.java | 3 ++-
thirdparty/vars.sh | 8 ++++----
6 files changed, 39 insertions(+), 9 deletions(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 02/02: [improvement](orc)improve hdfs scan performance (#11501)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit d524c813bd7c67a53c39385e2786cbe7266c7539
Author: dujl <du...@bytedance.com>
AuthorDate: Thu Aug 4 17:10:30 2022 +0800
[improvement](orc)improve hdfs scan performance (#11501)
Co-authored-by: root <ro...@bytedance>
---
fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java | 1 +
thirdparty/vars.sh | 8 ++++----
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index a7772b1aee..1c5daa9770 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -20,6 +20,7 @@ package org.apache.doris.common.util;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
import org.apache.doris.qe.ConnectContext;
import org.apache.logging.log4j.LogManager;
diff --git a/thirdparty/vars.sh b/thirdparty/vars.sh
index 8d9ce00628..de005c6dbb 100755
--- a/thirdparty/vars.sh
+++ b/thirdparty/vars.sh
@@ -345,10 +345,10 @@ KRB5_SOURCE="krb5-1.19"
KRB5_MD5SUM="aaf18447a5a014aa3b7e81814923f4c9"
# hdfs3
-HDFS3_DOWNLOAD="https://github.com/yangzhg/libhdfs3/archive/refs/tags/v2.3.1.tar.gz"
-HDFS3_NAME="libhdfs3-2.3.1.tar.gz"
-HDFS3_SOURCE="libhdfs3-2.3.1"
-HDFS3_MD5SUM="64ab3004826d83b23522ccf26940db94"
+HDFS3_DOWNLOAD="https://github.com/apache/doris-thirdparty/archive/refs/tags/libhdfs3-v2.3.2.tar.gz"
+HDFS3_NAME="doris-thirdparty-libhdfs3-v2.3.2.tar.gz"
+HDFS3_SOURCE="doris-thirdparty-libhdfs3-v2.3.2"
+HDFS3_MD5SUM="5087ffec0fda4fbcd60a53ed92eb4d2d"
#libdivide
LIBDIVIDE_DOWNLOAD="https://github.com/ridiculousfish/libdivide/archive/5.0.tar.gz"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org
[doris] 01/02: [multi-catalog](oss)Support hive external table on Ali oss. (#11489)
Posted by yi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch dev-1.1.2
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4a1d24b62f9cd184d52baf464698044cfb2cb3b5
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Thu Aug 4 17:44:34 2022 +0800
[multi-catalog](oss)Support hive external table on Ali oss. (#11489)
---
.../doris/catalog/HiveMetaStoreClientHelper.java | 18 +++++++++++++++---
.../java/org/apache/doris/catalog/IcebergTable.java | 3 ++-
.../src/main/java/org/apache/doris/common/Config.java | 6 ++++++
.../main/java/org/apache/doris/common/util/Util.java | 9 +++++++++
.../java/org/apache/doris/planner/HiveScanNode.java | 3 ++-
5 files changed, 34 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 0d5bcf9671..47fe69111d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -31,6 +31,7 @@ import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.SlotRef;
import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.analysis.StringLiteral;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.util.BrokerUtil;
import org.apache.doris.thrift.TBrokerFileStatus;
@@ -251,7 +252,7 @@ public class HiveMetaStoreClientHelper {
throws DdlException {
List<RemoteIterator<LocatedFileStatus>> allIterators = new ArrayList<>();
for (Partition p : partitions) {
- String location = p.getSd().getLocation();
+ String location = normalizeS3LikeSchema(p.getSd().getLocation());
Path path = new Path(location);
allIterators.addAll(getRemoteIterator(path, configuration, properties, isSecurityEnabled));
}
@@ -261,7 +262,7 @@ public class HiveMetaStoreClientHelper {
// Get remote iterators for given table
private static List<RemoteIterator<LocatedFileStatus>> getRemoteIterator(Table table, Configuration configuration,
boolean isSecurityEnabled, Map<String, String> properties, boolean onS3) throws DdlException {
- String location = table.getSd().getLocation();
+ String location = normalizeS3LikeSchema(table.getSd().getLocation());
Path path = new Path(location);
return getRemoteIterator(path, configuration, properties, isSecurityEnabled);
}
@@ -280,12 +281,23 @@ public class HiveMetaStoreClientHelper {
FileSystem fileSystem = path.getFileSystem(conf);
iterators.add(fileSystem.listLocatedStatus(path));
} catch (IOException e) {
- LOG.warn("Get HDFS file remote iterator failed. {}" + e.getMessage());
+ LOG.warn("Get HDFS file remote iterator failed. {}", e.getMessage());
throw new DdlException("Get HDFS file remote iterator failed. Error: " + e.getMessage());
}
return iterators;
}
+ public static String normalizeS3LikeSchema(String location) {
+ String[] objectStorages = Config.s3_compatible_object_storages.split(",");
+ for (String objectStorage : objectStorages) {
+ if (location.startsWith(objectStorage + "://")) {
+ location = location.replaceFirst(objectStorage, "s3");
+ break;
+ }
+ }
+ return location;
+ }
+
private static String getAllFileStatus(List<TBrokerFileStatus> fileStatuses,
List<RemoteIterator<LocatedFileStatus>> remoteIterators, Configuration configuration,
boolean isSecurityEnabled, Map<String, String> properties, boolean onS3) throws DdlException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java
index 41a0458f4e..7fbabd38b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java
@@ -20,6 +20,7 @@ package org.apache.doris.catalog;
import org.apache.doris.analysis.StorageBackend;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.Text;
+import org.apache.doris.common.util.Util;
import org.apache.doris.external.iceberg.IcebergCatalog;
import org.apache.doris.external.iceberg.IcebergCatalogMgr;
import org.apache.doris.thrift.TBrokerFileStatus;
@@ -130,7 +131,7 @@ public class IcebergTable extends Table {
// analyze storage type
String storagePrefix = strings[0].split(":")[0];
- if (storagePrefix.equalsIgnoreCase("s3")) {
+ if (Util.isS3CompatibleStorageSchema(storagePrefix)) {
this.storageType = StorageBackend.StorageType.S3;
} else if (storagePrefix.equalsIgnoreCase("hdfs")) {
this.storageType = StorageBackend.StorageType.HDFS;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
index c267f76fcc..3a51f7d0dc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/Config.java
@@ -1613,4 +1613,10 @@ public class Config extends ConfigBase {
@ConfField(mutable = false, masterOnly = true)
public static int backend_rpc_timeout_ms = 60000; // 1 min
+
+ /**
+ * List of S3 API compatible object storage systems.
+ */
+ @ConfField
+ public static String s3_compatible_object_storages = "s3,oss,cos,bos";
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
index 7cbc66c5a6..a7772b1aee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/Util.java
@@ -455,5 +455,14 @@ public class Util {
}
return s;
}
+
+ public static boolean isS3CompatibleStorageSchema(String schema) {
+ for (String objectStorage : Config.s3_compatible_object_storages.split(",")) {
+ if (objectStorage.equalsIgnoreCase(schema)) {
+ return true;
+ }
+ }
+ return false;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java
index cfd76de134..27ff3d036d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveScanNode.java
@@ -28,6 +28,7 @@ import org.apache.doris.catalog.HiveMetaStoreClientHelper;
import org.apache.doris.catalog.HiveTable;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.doris.common.util.Util;
import org.apache.doris.load.BrokerFileGroup;
import org.apache.doris.thrift.TBrokerFileStatus;
import org.apache.doris.thrift.TExplainLevel;
@@ -132,7 +133,7 @@ public class HiveScanNode extends BrokerScanNode {
private void setStorageType(String location) throws UserException {
String[] strings = StringUtils.split(location, "/");
String storagePrefix = strings[0].split(":")[0];
- if (storagePrefix.equalsIgnoreCase("s3")) {
+ if (Util.isS3CompatibleStorageSchema(storagePrefix)) {
this.storageType = StorageBackend.StorageType.S3;
} else if (storagePrefix.equalsIgnoreCase("hdfs")) {
this.storageType = StorageBackend.StorageType.HDFS;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org