You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/07/20 11:04:04 UTC
[doris] branch master updated: [data lake]Support hdfs ha for Iceberg table. (#11002)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6aadee9a2e [data lake]Support hdfs ha for Iceberg table. (#11002)
6aadee9a2e is described below
commit 6aadee9a2e3cca2af7e8906a04f0d5fe74aac839
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Jul 20 19:03:58 2022 +0800
[data lake]Support hdfs ha for Iceberg table. (#11002)
* Support Iceberg on HDFS with HA mode enabled.
---
.../ecosystem/external-table/iceberg-of-doris.md | 37 ++++++++++++++++++++--
.../ecosystem/external-table/iceberg-of-doris.md | 37 ++++++++++++++++++++--
.../org/apache/doris/catalog/IcebergProperty.java | 21 ++++++++++++
.../org/apache/doris/catalog/IcebergTable.java | 7 ++--
.../doris/external/iceberg/IcebergCatalogMgr.java | 11 +++++++
5 files changed, 105 insertions(+), 8 deletions(-)
diff --git a/docs/en/docs/ecosystem/external-table/iceberg-of-doris.md b/docs/en/docs/ecosystem/external-table/iceberg-of-doris.md
index 62cbac8c4d..0e0090c8ea 100644
--- a/docs/en/docs/ecosystem/external-table/iceberg-of-doris.md
+++ b/docs/en/docs/ecosystem/external-table/iceberg-of-doris.md
@@ -62,7 +62,7 @@ Iceberg tables can be created in Doris in two ways. You do not need to declare t
);
- -- Example: Mount iceberg_table under iceberg_db in Iceberg
+ -- Example 1: Mount iceberg_table under iceberg_db in Iceberg
CREATE TABLE `t_iceberg`
ENGINE = ICEBERG
PROPERTIES (
@@ -70,6 +70,21 @@ Iceberg tables can be created in Doris in two ways. You do not need to declare t
"iceberg.table" = "iceberg_table",
"iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
"iceberg.catalog.type" = "HIVE_CATALOG"
+
+
+ -- Example 2: Mount iceberg_table under iceberg_db in Iceberg, with HDFS HA enabled.
+ CREATE TABLE `t_iceberg`
+ ENGINE = ICEBERG
+ PROPERTIES (
+ "iceberg.database" = "iceberg_db",
+ "iceberg.table" = "iceberg_table"
+ "iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
+ "iceberg.catalog.type" = "HIVE_CATALOG",
+ "dfs.nameservices"="HDFS8000463",
+ "dfs.ha.namenodes.HDFS8000463"="nn2,nn1",
+ "dfs.namenode.rpc-address.HDFS8000463.nn2"="172.21.16.5:4007",
+ "dfs.namenode.rpc-address.HDFS8000463.nn1"="172.21.16.26:4007",
+ "dfs.client.failover.proxy.provider.HDFS8000463"="org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
);
```
@@ -115,7 +130,7 @@ You can also create an Iceberg table by explicitly specifying the column definit
"iceberg.catalog.type" = "HIVE_CATALOG"
);
- -- Example: Mount iceberg_table under iceberg_db in Iceberg
+ -- Example 1: Mount iceberg_table under iceberg_db in Iceberg
CREATE TABLE `t_iceberg` (
`id` int NOT NULL COMMENT "id number",
`name` varchar(10) NOT NULL COMMENT "user name"
@@ -126,6 +141,24 @@ You can also create an Iceberg table by explicitly specifying the column definit
"iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
"iceberg.catalog.type" = "HIVE_CATALOG"
);
+
+ -- Example 2: Mount iceberg_table under iceberg_db in Iceberg, with HDFS HA enabled.
+ CREATE TABLE `t_iceberg` (
+ `id` int NOT NULL COMMENT "id number",
+ `name` varchar(10) NOT NULL COMMENT "user name"
+ ) ENGINE = ICEBERG
+ PROPERTIES (
+ "iceberg.database" = "iceberg_db",
+ "iceberg.table" = "iceberg_table",
+ "iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
+ "iceberg.catalog.type" = "HIVE_CATALOG",
+ "dfs.nameservices"="HDFS8000463",
+ "dfs.ha.namenodes.HDFS8000463"="nn2,nn1",
+ "dfs.namenode.rpc-address.HDFS8000463.nn2"="172.21.16.5:4007",
+ "dfs.namenode.rpc-address.HDFS8000463.nn1"="172.21.16.26:4007",
+ "dfs.client.failover.proxy.provider.HDFS8000463"="org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
+
```
#### Parameter Description
diff --git a/docs/zh-CN/docs/ecosystem/external-table/iceberg-of-doris.md b/docs/zh-CN/docs/ecosystem/external-table/iceberg-of-doris.md
index 146c59aee4..dd231ed589 100644
--- a/docs/zh-CN/docs/ecosystem/external-table/iceberg-of-doris.md
+++ b/docs/zh-CN/docs/ecosystem/external-table/iceberg-of-doris.md
@@ -62,7 +62,7 @@ Iceberg External Table of Doris 提供了 Doris 直接访问 Iceberg 外部表
);
- -- 例子：挂载 Iceberg 中 iceberg_db 下的 iceberg_table
+ -- 例子1：挂载 Iceberg 中 iceberg_db 下的 iceberg_table
CREATE TABLE `t_iceberg`
ENGINE = ICEBERG
PROPERTIES (
@@ -71,6 +71,22 @@ Iceberg External Table of Doris 提供了 Doris 直接访问 Iceberg 外部表
"iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
"iceberg.catalog.type" = "HIVE_CATALOG"
);
+
+
+ -- 例子2：挂载 Iceberg 中 iceberg_db 下的 iceberg_table，HDFS开启HA
+ CREATE TABLE `t_iceberg`
+ ENGINE = ICEBERG
+ PROPERTIES (
+ "iceberg.database" = "iceberg_db",
+ "iceberg.table" = "iceberg_table",
+ "iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
+ "iceberg.catalog.type" = "HIVE_CATALOG",
+ "dfs.nameservices"="HDFS8000463",
+ "dfs.ha.namenodes.HDFS8000463"="nn2,nn1",
+ "dfs.namenode.rpc-address.HDFS8000463.nn2"="172.21.16.5:4007",
+ "dfs.namenode.rpc-address.HDFS8000463.nn1"="172.21.16.26:4007",
+ "dfs.client.failover.proxy.provider.HDFS8000463"="org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
```
2. 创建一个 Iceberg 数据库，用于挂载远端对应 Iceberg 数据库，同时挂载该 database 下的所有 table。
@@ -114,7 +130,7 @@ Iceberg External Table of Doris 提供了 Doris 直接访问 Iceberg 外部表
"iceberg.catalog.type" = "HIVE_CATALOG"
);
- -- 例子：挂载 Iceberg 中 iceberg_db 下的 iceberg_table
+ -- 例子1：挂载 Iceberg 中 iceberg_db 下的 iceberg_table
CREATE TABLE `t_iceberg` (
`id` int NOT NULL COMMENT "id number",
`name` varchar(10) NOT NULL COMMENT "user name"
@@ -125,6 +141,23 @@ Iceberg External Table of Doris 提供了 Doris 直接访问 Iceberg 外部表
"iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
"iceberg.catalog.type" = "HIVE_CATALOG"
);
+
+ -- 例子2：挂载 Iceberg 中 iceberg_db 下的 iceberg_table，HDFS开启HA
+ CREATE TABLE `t_iceberg` (
+ `id` int NOT NULL COMMENT "id number",
+ `name` varchar(10) NOT NULL COMMENT "user name"
+ ) ENGINE = ICEBERG
+ PROPERTIES (
+ "iceberg.database" = "iceberg_db",
+ "iceberg.table" = "iceberg_table",
+ "iceberg.hive.metastore.uris" = "thrift://192.168.0.1:9083",
+ "iceberg.catalog.type" = "HIVE_CATALOG",
+ "dfs.nameservices"="HDFS8000463",
+ "dfs.ha.namenodes.HDFS8000463"="nn2,nn1",
+ "dfs.namenode.rpc-address.HDFS8000463.nn2"="172.21.16.5:4007",
+ "dfs.namenode.rpc-address.HDFS8000463.nn1"="172.21.16.26:4007",
+ "dfs.client.failover.proxy.provider.HDFS8000463"="org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider"
+ );
```
#### 参数说明：
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java
index ffc933eb7a..882524a575 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergProperty.java
@@ -17,6 +17,9 @@
package org.apache.doris.catalog;
+import com.google.common.collect.Maps;
+
+import java.util.Iterator;
import java.util.Map;
/**
@@ -27,6 +30,7 @@ public class IcebergProperty {
public static final String ICEBERG_TABLE = "iceberg.table";
public static final String ICEBERG_HIVE_METASTORE_URIS = "iceberg.hive.metastore.uris";
public static final String ICEBERG_CATALOG_TYPE = "iceberg.catalog.type";
+ public static final String ICEBERG_HDFS_PREFIX = "dfs";
private boolean exist;
@@ -34,6 +38,17 @@ public class IcebergProperty {
private String table;
private String hiveMetastoreUris;
private String catalogType;
+ private Map<String, String> dfsProperties = Maps.newHashMap();
+
+ private void initDfsProperties(Map<String, String> properties) {
+ Iterator<Map.Entry<String, String>> iterator = properties.entrySet().iterator();
+ while (iterator.hasNext()) {
+ Map.Entry<String, String> entry = iterator.next();
+ if (entry.getKey().startsWith(ICEBERG_HDFS_PREFIX)) {
+ dfsProperties.put(entry.getKey(), entry.getValue());
+ }
+ }
+ }
public IcebergProperty(Map<String, String> properties) {
if (properties != null && !properties.isEmpty()) {
@@ -42,6 +57,7 @@ public class IcebergProperty {
this.table = properties.get(ICEBERG_TABLE);
this.hiveMetastoreUris = properties.get(ICEBERG_HIVE_METASTORE_URIS);
this.catalogType = properties.get(ICEBERG_CATALOG_TYPE);
+ initDfsProperties(properties);
} else {
this.exist = false;
}
@@ -54,6 +70,7 @@ public class IcebergProperty {
this.table = otherProperty.table;
this.hiveMetastoreUris = otherProperty.hiveMetastoreUris;
this.catalogType = otherProperty.catalogType;
+ this.dfsProperties = otherProperty.dfsProperties;
}
public boolean isExist() {
@@ -83,4 +100,8 @@ public class IcebergProperty {
public void setTable(String table) {
this.table = table;
}
+
+ public Map<String, String> getDfsProperties() {
+ return dfsProperties;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java
index eb748ed807..685602b790 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/IcebergTable.java
@@ -86,10 +86,9 @@ public class IcebergTable extends Table {
this.icebergDb = icebergProperty.getDatabase();
this.icebergTbl = icebergProperty.getTable();
- icebergProperties.put(IcebergProperty.ICEBERG_HIVE_METASTORE_URIS,
- icebergProperty.getHiveMetastoreUris());
- icebergProperties.put(IcebergProperty.ICEBERG_CATALOG_TYPE,
- icebergProperty.getCatalogType());
+ icebergProperties.put(IcebergProperty.ICEBERG_HIVE_METASTORE_URIS, icebergProperty.getHiveMetastoreUris());
+ icebergProperties.put(IcebergProperty.ICEBERG_CATALOG_TYPE, icebergProperty.getCatalogType());
+ icebergProperties.putAll(icebergProperty.getDfsProperties());
this.icebergTable = icebergTable;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergCatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergCatalogMgr.java
index 322140fb65..c219c0fade 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergCatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/iceberg/IcebergCatalogMgr.java
@@ -35,6 +35,7 @@ import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -126,6 +127,16 @@ public class IcebergCatalogMgr {
copiedProps.remove(IcebergProperty.ICEBERG_TABLE);
}
+ if (!copiedProps.isEmpty()) {
+ Iterator<Map.Entry<String, String>> iter = copiedProps.entrySet().iterator();
+ while (iter.hasNext()) {
+ Map.Entry<String, String> entry = iter.next();
+ if (entry.getKey().startsWith(IcebergProperty.ICEBERG_HDFS_PREFIX)) {
+ iter.remove();
+ }
+ }
+ }
+
if (!copiedProps.isEmpty()) {
throw new DdlException("Unknown table properties: " + copiedProps.toString());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org