You are viewing a plain text version of this content. The canonical link for it is here.
Posted to submarine-dev@hadoop.apache.org by li...@apache.org on 2019/10/06 13:29:32 UTC
[hadoop-submarine] branch master updated: SUBMARINE-209. Run
workbench server in mini submarine.
This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-submarine.git
The following commit(s) were added to refs/heads/master by this push:
new c8f1fa9 SUBMARINE-209. Run workbench server in mini submarine.
c8f1fa9 is described below
commit c8f1fa9d0c51622567ee4eec2548bd96c351f2d2
Author: Zac Zhou <yu...@gmail.com>
AuthorDate: Mon Sep 30 23:52:30 2019 +0800
SUBMARINE-209. Run workbench server in mini submarine.
### What is this PR for?
It's needed to run workbench server in mini-submarine.
It's easier for beginners to use submarine.
### What type of PR is it?
Improvement
### Todos
* [ ] - Task
### What is the Jira issue?
https://issues.apache.org/jira/browse/SUBMARINE-209
### How should this be tested?
https://travis-ci.org/yuanzac/hadoop-submarine/builds/591568607
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? Yes
* Does this needs documentation? Yes
Author: Zac Zhou <yu...@gmail.com>
Closes #27 from yuanzac/topic/SUBMARINE-209 and squashes the following commits:
8b71ff9 [Zac Zhou] SUBMARINE-209. Run workbench server in mini submarine.
---
.gitignore | 1 +
dev-support/mini-submarine/Dockerfile | 6 ++++-
dev-support/mini-submarine/README.md | 16 ++++++++++---
dev-support/mini-submarine/build_mini-submarine.sh | 22 ++++++++++++++---
dev-support/mini-submarine/conf/setup-mysql.sh | 28 ++++++++++++++++++++++
5 files changed, 66 insertions(+), 7 deletions(-)
diff --git a/.gitignore b/.gitignore
index 3ffe165..01f21b7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -57,6 +57,7 @@ venv.bak/
*.tar.gz
*.jar
target
+dev-support/mini-submarine/database
submodules/tony/tony-core/build
submarine-workbench/workbench-web/node
submarine-workbench/workbench-web/node_modules
diff --git a/dev-support/mini-submarine/Dockerfile b/dev-support/mini-submarine/Dockerfile
index 743f07e..70c5128 100644
--- a/dev-support/mini-submarine/Dockerfile
+++ b/dev-support/mini-submarine/Dockerfile
@@ -74,6 +74,8 @@ EXPOSE 8030 8031 8032 8033 8040 8042 8088
EXPOSE 2181 2888 3888
#Other ports
EXPOSE 49707 2122
+# Workbench port
+EXPOSE 8080
#Add spark dynamic allocation jar
#ADD spark-2.4.0-yarn-shuffle.jar /usr/local/hadoop/share/hadoop/yarn/spark-2.4.0-yarn-shuffle.jar
@@ -113,8 +115,10 @@ ENV IMAGE_N=${IMAGE_NAME}
# Install Submarine
ARG SUBMARINE_VERSION=
ENV SUBMARINE_VER=${SUBMARINE_VERSION}
-ADD hadoop-submarine-${SUBMARINE_VER}.tar.gz /opt
+ADD submarine-dist-${SUBMARINE_VER}*.tar.gz /opt
+RUN ln -s /opt/submarine-dist-${SUBMARINE_VER}* /opt/submarine-current
ADD submarine /home/yarn/submarine
+ADD database /home/yarn/database
# Build virtual python env
RUN cd /home/yarn/submarine && \
diff --git a/dev-support/mini-submarine/README.md b/dev-support/mini-submarine/README.md
index e0d77a6..d83a3e1 100644
--- a/dev-support/mini-submarine/README.md
+++ b/dev-support/mini-submarine/README.md
@@ -39,7 +39,7 @@ docker pull hadoopsubmarine/mini-submarine:0.2.0
#### Run mini-submarine image
```
-docker run -it -h submarine-dev --net=bridge --privileged local/mini-submarine:0.2.0 /bin/bash
+docker run -it -h submarine-dev --net=bridge --privileged -P local/mini-submarine:0.2.0 /bin/bash
# In the container, use root user to bootstrap hdfs and yarn
/tmp/hadoop-config/bootstrap.sh
@@ -70,6 +70,17 @@ hdfs dfs -ls /user
> drwxr-xr-x - yarn supergroup 0 2019-07-22 07:59 /user/yarn
+### Run workbench server
+
+1. Setup mysql mariadb server
+```
+/tmp/hadoop-config/setup-mysql.sh
+```
+
+2. Start workbench server
+```
+/opt/submarine-current/bin/workbench-daemon.sh start getMysqlJar
+```
### Run a sumbarine job
@@ -232,7 +243,6 @@ Spark jobs are supported as well.
cd && cd spark-script && ./run_spark.sh
```
-
## Question and answer
1. Submarine package name error
@@ -246,4 +256,4 @@ cd && cd spark-script && ./run_spark.sh
<value>org.apache.submarine.runtimes.tony.TonyRuntimeFactory</value>
</property>
</configuration>
- ```
\ No newline at end of file
+ ```
diff --git a/dev-support/mini-submarine/build_mini-submarine.sh b/dev-support/mini-submarine/build_mini-submarine.sh
index 6b2aafc..198a815 100755
--- a/dev-support/mini-submarine/build_mini-submarine.sh
+++ b/dev-support/mini-submarine/build_mini-submarine.sh
@@ -16,9 +16,16 @@
hadoop_v=2.9.2
spark_v=2.4.4
-submarine_v=0.2.0
+submarine_v=0.3.0-SNAPSHOT
image_name="local/mini-submarine:${submarine_v}"
+if [ -L ${BASH_SOURCE-$0} ]; then
+ PWD=$(dirname $(readlink "${BASH_SOURCE-$0}"))
+else
+ PWD=$(dirname ${BASH_SOURCE-$0})
+fi
+export MINI_PATH=$(cd "${PWD}">/dev/null; pwd)
+
download_package() {
if [ -f "$1" ]; then
echo "Found $1"
@@ -42,8 +49,17 @@ download_package "hadoop-${hadoop_v}.tar.gz" "http://mirrors.tuna.tsinghua.edu.c
download_package "spark-${spark_v}-bin-hadoop2.7.tgz" "http://mirrors.tuna.tsinghua.edu.cn/apache/spark/spark-${spark_v}"
# download zookeeper
download_package "zookeeper-3.4.14.tar.gz" "http://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.4.14"
-# download submarine
-download_package "hadoop-submarine-${submarine_v}.tar.gz" "http://mirror.bit.edu.cn/apache/hadoop/submarine/submarine-${submarine_v}"
+
+cd ../..
+mysql_connector_exists=$(find -L "submarine-dist/target" -name "submarine-dist-${submarine_v}*.tar.gz")
+# Build source code if the package doesn't exist.
+if [[ -z "${mysql_connector_exists}" ]]; then
+ mvn clean package -DskipTests
+fi
+
+cp submarine-dist/target/submarine-dist-${submarine_v}*.tar.gz ${MINI_PATH}
+cp -r docs/database ${MINI_PATH}
+cd ${MINI_PATH}
# build image
echo "Start building the mini-submarine docker image..."
diff --git a/dev-support/mini-submarine/conf/setup-mysql.sh b/dev-support/mini-submarine/conf/setup-mysql.sh
new file mode 100755
index 0000000..7156a1c
--- /dev/null
+++ b/dev-support/mini-submarine/conf/setup-mysql.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install mariadb
+apt-get -y install mariadb-server
+service mysql start
+mysql -e "CREATE DATABASE submarineDB_test;"
+mysql -e "CREATE USER 'submarine_test'@'%' IDENTIFIED BY 'password_test';"
+mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'submarine_test'@'%';"
+mysql -e "use submarineDB_test; source /home/yarn/database/submarine.sql; source /home/yarn/database/submarine-data.sql;"
+
+mysql -e "CREATE DATABASE submarineDB;"
+mysql -e "CREATE USER 'submarine'@'%' IDENTIFIED BY 'password';"
+mysql -e "GRANT ALL PRIVILEGES ON * . * TO 'submarine'@'%';"
+mysql -e "use submarineDB; source /home/yarn/database/submarine.sql; source /home/yarn/database/submarine-data.sql;"