You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2024/02/19 09:24:10 UTC

(doris) 10/18: [enhancement](paimon)add obs filesystem for paimon (#31061)

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git

commit bda2042c7824073596f5f22bb80189b45b51cddf
Author: wuwenchi <wu...@hotmail.com>
AuthorDate: Mon Feb 19 10:09:41 2024 +0800

    [enhancement](paimon)add obs filesystem for paimon (#31061)
---
 docs/en/docs/lakehouse/multi-catalog/paimon.md       | 20 ++++++++++++++++++++
 docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md    | 20 ++++++++++++++++++++
 .../datasource/paimon/PaimonFileExternalCatalog.java | 10 ++++++++++
 3 files changed, 50 insertions(+)

diff --git a/docs/en/docs/lakehouse/multi-catalog/paimon.md b/docs/en/docs/lakehouse/multi-catalog/paimon.md
index c4c7bb64860..49012cf7318 100644
--- a/docs/en/docs/lakehouse/multi-catalog/paimon.md
+++ b/docs/en/docs/lakehouse/multi-catalog/paimon.md
@@ -93,6 +93,26 @@ CREATE CATALOG `paimon_s3` PROPERTIES (
 );
 ```
 
+#### OBS
+
+> Note that.
+>
+> user need download [paimon-s3-0.6.0-incubating.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/0.6.0-incubating/paimon-s3-0.6.0-incubating.jar)
+>
+> Place it in directory `${DORIS_HOME}/be/lib/java_extensions/preload-extensions` and restart be
+>
+> Starting from version 2.0.2, this file can be placed in BE's `custom_lib/` directory (if it does not exist, just create it manually) to prevent the file from being lost due to the replacement of the lib directory when upgrading the cluster.
+
+```sql
+CREATE CATALOG `paimon_obs` PROPERTIES (
+    "type" = "paimon",
+    "warehouse" = "obs://bucket_name/paimon",
+    "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com",
+    "obs.access_key"="ak",
+    "obs.secret_key"="sk"
+);
+```
+
 #### COS
 
 > Note that.
diff --git a/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md b/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md
index 79632be6772..d196df7a87e 100644
--- a/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md
+++ b/docs/zh-CN/docs/lakehouse/multi-catalog/paimon.md
@@ -93,6 +93,26 @@ CREATE CATALOG `paimon_s3` PROPERTIES (
 );
 ```
 
+#### OBS
+
+> 注意:
+>
+> 用户需要手动下载[paimon-s3-0.6.0-incubating.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-s3/0.6.0-incubating/paimon-s3-0.6.0-incubating.jar)
+
+> 放在 `${DORIS_HOME}/be/lib/java_extensions/preload-extensions` 目录下并重启be。
+>
+> 从 2.0.2 版本起,可以将这个文件放置在BE的 `custom_lib/` 目录下(如不存在,手动创建即可),以防止升级集群时因为 lib 目录被替换而导致文件丢失。
+
+```sql
+CREATE CATALOG `paimon_obs` PROPERTIES (
+    "type" = "paimon",
+    "warehouse" = "obs://bucket_name/paimon",
+    "obs.endpoint"="obs.cn-north-4.myhuaweicloud.com",
+    "obs.access_key"="ak",
+    "obs.secret_key"="sk"
+);
+```
+
 #### COS
 
 > 注意:
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
index 9e8fa2dc455..62b29c98f3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.paimon;
 import org.apache.doris.datasource.CatalogProperty;
 import org.apache.doris.datasource.property.PropertyConverter;
 import org.apache.doris.datasource.property.constants.CosProperties;
+import org.apache.doris.datasource.property.constants.ObsProperties;
 import org.apache.doris.datasource.property.constants.PaimonProperties;
 
 import org.apache.logging.log4j.LogManager;
@@ -68,6 +69,15 @@ public class PaimonFileExternalCatalog extends PaimonExternalCatalog {
                     properties.get(CosProperties.SECRET_KEY));
             options.put(PaimonProperties.WAREHOUSE,
                     options.get(PaimonProperties.WAREHOUSE).replace("cosn://", "s3://"));
+        } else if (properties.containsKey(ObsProperties.ENDPOINT)) {
+            options.put(PaimonProperties.PAIMON_S3_ENDPOINT,
+                    properties.get(ObsProperties.ENDPOINT));
+            options.put(PaimonProperties.PAIMON_S3_ACCESS_KEY,
+                    properties.get(ObsProperties.ACCESS_KEY));
+            options.put(PaimonProperties.PAIMON_S3_SECRET_KEY,
+                    properties.get(ObsProperties.SECRET_KEY));
+            options.put(PaimonProperties.WAREHOUSE,
+                    options.get(PaimonProperties.WAREHOUSE).replace("obs://", "s3://"));
         }
 
         if (properties.containsKey(PropertyConverter.USE_PATH_STYLE)) {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org