You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/06 07:25:06 UTC
[doris] branch master updated: [feature](docker)Docker example hive-broker-doris (#20473)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1b02b28c40 [feature](docker)Docker example hive-broker-doris (#20473)
1b02b28c40 is described below
commit 1b02b28c40ab66a49643158384902dcbebf68906
Author: FreeOnePlus <54...@users.noreply.github.com>
AuthorDate: Tue Jun 6 15:24:58 2023 +0800
[feature](docker)Docker example hive-broker-doris (#20473)
add new docker example: hdfs-broker-doris
---
.../example/hdfs-broker-doris/broker-init.sql | 29 +++++++
.../example/hdfs-broker-doris/docker-compose.yaml | 97 ++++++++++++++++++++++
.../example/hdfs-broker-doris/doris-init.sql | 27 ++++++
.../example/hdfs-broker-doris/hdfs-init.sql | 18 ++++
.../example/hdfs-broker-doris/user_basic_data.sql | 20 +++++
5 files changed, 191 insertions(+)
diff --git a/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/broker-init.sql b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/broker-init.sql
new file mode 100644
index 0000000000..d9926bb09b
--- /dev/null
+++ b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/broker-init.sql
@@ -0,0 +1,29 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+LOAD LABEL test.t1_20000000 (
+ DATA INFILE("hdfs://xxx.xxx.xxx.xxx:8020/user/doris/user_basic_data")
+ INTO TABLE `user_basic`
+ COLUMNS TERMINATED BY "," (user_id,user_name,user_msg)
+)
+with broker bk1 (
+ "fs.defaultFS"="hdfs://xxx.xxx.xxx.xxx:8020",
+ "hadoop.username"="hdfsuser"
+)
+PROPERTIES (
+ "timeout"="1200",
+ "max_filter_ratio"="0.1"
+);
diff --git a/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/docker-compose.yaml b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/docker-compose.yaml
new file mode 100644
index 0000000000..c86786b6f2
--- /dev/null
+++ b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/docker-compose.yaml
@@ -0,0 +1,97 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+version: "3"
+services:
+ namenode:
+ image: singularities/hadoop:2.8
+ volumes:
+ - /data:/data
+ command: start-hadoop namenode
+ hostname: namenode
+ environment:
+ HDFS_USER: hdfsuser
+ ports:
+ - "8020:8020"
+ - "14000:14000"
+ - "50070:50070"
+ - "50075:50075"
+ - "10020:10020"
+ - "13562:13562"
+ - "19888:19888"
+ datanode:
+ image: singularities/hadoop:2.8
+ hostname: datanode
+ volumes:
+ - /data/hdfs/:/data
+ command: start-hadoop datanode namenode
+ environment:
+ HDFS_USER: hdfsuser
+ ports:
+ - "50010:50010"
+ links:
+ - namenode
+
+ fe:
+ image: apache/doris:1.2.2-fe-x86_64
+ hostname: doris-fe
+ volumes:
+ - /data/fe/doris-meta/:/opt/apache-doris/fe/doris-meta/
+ - /data/fe/log/:/opt/apache-doris/fe/log/
+ environment:
+ - FE_SERVERS=fe1:172.20.80.1:9010
+ - FE_ID=1
+ restart: on-failure
+ networks:
+ doris_net:
+ ipv4_address: 172.20.80.1
+
+ be:
+ image: apache/doris:1.2.2-be-x86_64
+ hostname: doris-be
+ volumes:
+ - /data/be/storage/:/opt/apache-doris/be/storage/
+ - /data/be/log/:/opt/apache-doris/be/log/
+ environment:
+ - FE_SERVERS=fe1:172.20.80.1:9010
+ - BE_ADDR=172.20.80.2:9050
+ depends_on:
+ - fe
+ restart: on-failure
+ networks:
+ doris_net:
+ ipv4_address: 172.20.80.2
+
+ broker:
+ image: apache/doris:1.2.2-broker-x86_64
+ hostname: doris-broker
+ volumes:
+ - /data/broker/conf/:/opt/apache-doris/broker/conf/
+ - /data/broker/log/:/opt/apache-doris/broker/log/
+ environment:
+ - FE_SERVERS=fe1:172.20.80.1:9010
+ - BROKER_ADDR=bk1:172.20.80.3:9050
+ depends_on:
+ - fe
+ restart: on-failure
+ networks:
+ doris_net:
+ ipv4_address: 172.20.80.3
+networks:
+ doris_net:
+ ipam:
+ config:
+ - subnet: 172.20.80.0/24
diff --git a/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/doris-init.sql b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/doris-init.sql
new file mode 100644
index 0000000000..35347927c7
--- /dev/null
+++ b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/doris-init.sql
@@ -0,0 +1,27 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+create database test;
+use test;
+CREATE TABLE user_basic (
+ user_id int(11) NULL COMMENT "用户ID",
+ user_name varchar(20) NULL COMMENT "用户姓名",
+ user_msg varchar(20) NULL COMMENT "用户信息"
+) ENGINE=OLAP
+DUPLICATE KEY(user_id)
+COMMENT "OLAP"
+DISTRIBUTED BY HASH(user_id) BUCKETS 3
+PROPERTIES("replication_num" = "1");
diff --git a/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/hdfs-init.sql b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/hdfs-init.sql
new file mode 100644
index 0000000000..0da0b0d508
--- /dev/null
+++ b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/hdfs-init.sql
@@ -0,0 +1,18 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+hadoop fs -mkdir -p /user/doris
+hadoop fs -put /data/user_basic_data /user/doris/user_basic_data
diff --git a/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/user_basic_data.sql b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/user_basic_data.sql
new file mode 100644
index 0000000000..524d6bb4a5
--- /dev/null
+++ b/docker/runtime/docker-compose-demo/example/hdfs-broker-doris/user_basic_data.sql
@@ -0,0 +1,20 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+1,zhangsan,like doris
+2,lisi,like apache
+3,wangmazi,like apache doris
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org