You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ca...@apache.org on 2022/09/30 10:04:01 UTC
[incubator-linkis] branch dev-1.3.0 updated: [ISSUE-3578]Update build docker file (#3579)
This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new d364df15d [ISSUE-3578]Update build docker file (#3579)
d364df15d is described below
commit d364df15d4cfd28da3f19e448ac9d2d07645ef29
Author: aiceflower <ki...@sina.com>
AuthorDate: Fri Sep 30 18:03:56 2022 +0800
[ISSUE-3578]Update build docker file (#3579)
---
README.md | 3 ++-
README_CN.md | 9 +++++++--
linkis-dist/docker/ldh.Dockerfile | 21 +++++++++++++++++----
linkis-dist/docker/linkis.Dockerfile | 16 ++++++++++++++--
.../scripts/make-linikis-image-with-mysql-jdbc.sh | 7 +++++++
linkis-dist/docker/scripts/prepare-ldh-image.sh | 9 +++++++++
linkis-dist/docker/scripts/utils.sh | 5 +++--
linkis-dist/helm/README.md | 2 +-
linkis-dist/helm/README_CN.md | 2 +-
.../templates/linkis-cg-engineconnmanager.yaml | 4 ++++
10 files changed, 65 insertions(+), 13 deletions(-)
diff --git a/README.md b/README.md
index 7786edf90..c7047c3c6 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,8 @@ To solve this problem, we provide a script which can help to creating an custom
linkis image by yourself, the image created by this tool will be tagged as `linkis:with-jdbc` by default.
```shell
-$> LINKIS_IMAGE=linkis:1.3.0 ./linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
+$> LINKIS_IMAGE=linkis:1.3.0
+$> ./linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
# build dir: ...
# base image: linkis:1.3.0
diff --git a/README_CN.md b/README_CN.md
index f73d6a014..2846df54e 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -126,8 +126,11 @@ Linkis 自 2019 年开源发布以来，已累计积累了 700 多家试验企
>- [后端编译打包 ](https://linkis.apache.org/zh-CN/docs/latest/development/linkis-compile-and-package)
>- [管理台编译 ](https://linkis.apache.org/zh-CN/docs/latest/development/web-build)
+
+
```shell script
## 后端编译
+注意:如果使用 -Dlinkis.build.web=true 来构建web镜像，需要首先执行 管理台 linkis-web 的编译
### Mac OS/Linux 系统
# 1. 首次编译，请执行如下命令
@@ -143,12 +146,13 @@ Linkis 自 2019 年开源发布以来，已累计积累了 700 多家试验企
# - 选项 3: 构建 Linkis 服务部署包和 Docker 镜像（包括 Web 控制台）
./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true
# - 选项 4: 仅构建 Linkis 服务部署包和 Docker 镜像（包括 Web 控制台和 LDH 镜像，LDH 镜像包换了多个预先配置和部署好的 Hadoop 组件，主要面向测试用途）
-./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true -Dlinkis.build.ldh=true
+./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true -Dlinkis.build.ldh=true -Dlinkis.build.with.jdbc=true
### Windows 系统
mvnw.cmd -N install
mvnw.cmd clean install -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
+
## 管理台编译
cd incubator-linkis/linkis-web
npm install
@@ -161,7 +165,8 @@ npm run build
我们提供了一个脚本，它可以帮助你快速的基于官方的 Linkis 镜像创建一个集成了MySQL JDBC 的自定义镜像。 这个工具创建的镜像默认的名称是 `linkis:with-jdbc`。
```shell
-$> LINKIS_IMAGE=linkis:1.3.0 ./linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
+$> LINKIS_IMAGE=linkis:1.3.0
+$> ./linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
# build dir: ...
# base image: linkis:1.3.0
diff --git a/linkis-dist/docker/ldh.Dockerfile b/linkis-dist/docker/ldh.Dockerfile
index dc6b9ee2b..f6404f3b5 100644
--- a/linkis-dist/docker/ldh.Dockerfile
+++ b/linkis-dist/docker/ldh.Dockerfile
@@ -38,13 +38,23 @@ ARG LINKIS_VERSION=0.0.0
RUN useradd -r -s /bin/bash -u 100001 -g root -G wheel hadoop
+# if you want to set specific yum repos conf file, you can put its at linkis-dist/docker/CentOS-Base.repo
+# and exec [COPY apache-linkis-*-incubating-bin/docker/CentOS-Epel.repo /etc/yum.repos.d/CentOS-Epel.repo]
+
+# TODO: remove install mysql client when schema-init-tools is ready
RUN yum install -y \
- vim unzip curl sudo krb5-workstation sssd crontabs net-tools python-pip \
+ less ls vim unzip curl sudo krb5-workstation sssd crontabs net-tools python-pip glibc-common \
java-${JDK_VERSION}-${JDK_BUILD_REVISION} \
java-${JDK_VERSION}-devel-${JDK_BUILD_REVISION} \
mysql \
&& yum clean all
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+ && localedef -c -f UTF-8 -i en_US en_US.UTF-8
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US:zh LC_TIME=en_US.UTF-8
+ENV TZ="Asia/Shanghai"
+
+
RUN sed -i "s#^%wheel.*#%wheel ALL=(ALL) NOPASSWD: ALL#g" /etc/sudoers
RUN mkdir -p /opt/ldh/${LINKIS_VERSION} \
@@ -70,9 +80,12 @@ RUN mkdir -p /etc/ldh \
&& ln -s /opt/ldh/${LINKIS_VERSION}/flink-${FLINK_VERSION} /opt/ldh/current/flink \
&& ln -s /opt/ldh/${LINKIS_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin /opt/ldh/current/zookeeper
-RUN curl -L -o /opt/ldh/current/hive/lib/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar \
- https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_JDBC_VERSION}/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar \
- && cp /opt/ldh/current/hive/lib/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar /opt/ldh/current/spark/jars/
+# after create soft link
+ADD ldh-tars/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar /opt/ldh/current/hive/lib/
+ADD ldh-tars/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar /opt/ldh/current/spark/jars/
+
+
+
ENV JAVA_HOME /etc/alternatives/jre
ENV PATH /opt/ldh/current/hadoop/bin:/opt/ldh/current/hive/bin:/opt/ldh/current/spark/bin:/opt/ldh/current/flink/bin:/opt/ldh/current/zookeeper/bin:$PATH
diff --git a/linkis-dist/docker/linkis.Dockerfile b/linkis-dist/docker/linkis.Dockerfile
index 62162ed18..83118cd8a 100644
--- a/linkis-dist/docker/linkis.Dockerfile
+++ b/linkis-dist/docker/linkis.Dockerfile
@@ -26,14 +26,23 @@ FROM ${IMAGE_BASE} as linkis-base
ARG JDK_VERSION=1.8.0-openjdk
ARG JDK_BUILD_REVISION=1.8.0.332.b09-1.el7_9
+# if you want to set specific yum repos conf file, you can put its at linkis-dist/docker/CentOS-Base.repo
+# and exec [COPY apache-linkis-*-incubating-bin/docker/CentOS-Epel.repo /etc/yum.repos.d/CentOS-Epel.repo]
+
# TODO: remove install mysql client when schema-init-tools is ready
RUN yum install -y \
- vim unzip curl sudo krb5-workstation sssd crontabs python-pip \
+ less ls vim unzip curl sudo krb5-workstation sssd crontabs python-pip glibc-common \
java-${JDK_VERSION}-${JDK_BUILD_REVISION} \
java-${JDK_VERSION}-devel-${JDK_BUILD_REVISION} \
mysql \
&& yum clean all
+RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime \
+ && localedef -c -f UTF-8 -i en_US en_US.UTF-8
+ENV LANG=en_US.UTF-8 LANGUAGE=en_US:zh LC_TIME=en_US.UTF-8
+ENV TZ="Asia/Shanghai"
+
+
######################################################################
# linkis image for release
@@ -62,9 +71,12 @@ ENV LINKIS_CONF_DIR ${LINKIS_CONF_DIR}
ENV LINKIS_CLIENT_CONF_DIR ${LINKIS_CONF_DIR}
ENV LINKIS_HOME ${LINKIS_HOME}
-ADD apache-linkis-${LINKIS_VERSION}-incubating-bin.tar.gz /opt/tmp/
+# can do some pre-operations
+ADD apache-linkis-${LINKIS_VERSION}-incubating-bin /opt/tmp/
RUN mv /opt/tmp/linkis-package/* ${LINKIS_HOME}/ \
+ && mv /opt/tmp/release-docs/LICENSE ${LINKIS_HOME}/LICENSE \
+ && mv /opt/tmp/release-docs/NOTICE ${LINKIS_HOME}/NOTICE \
&& rm -rf /opt/tmp
RUN chmod g+w -R ${LINKIS_HOME} && chown ${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LINKIS_HOME} \
diff --git a/linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh b/linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
index 1b722f41a..873329473 100755
--- a/linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
+++ b/linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
@@ -34,6 +34,13 @@ echo "# mysql jdbc version: ${MYSQL_JDBC_VERSION}"
download ${MYSQL_JDBC_URL} ${MYSQL_JDBC_FILENAME} ${BUILD_DIR}
+echo "try to exec: docker build -f ${WORK_DIR}/../linkis-with-mysql-jdbc.Dockerfile \
+ -t ${IMAGE_NAME} \
+ --build-arg LINKIS_IMAGE=${LINKIS_IMAGE} \
+ --build-arg LINKIS_HOME=${LINKIS_HOME} \
+ --build-arg MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION} \
+ ${BUILD_DIR}"
+
docker build -f ${WORK_DIR}/../linkis-with-mysql-jdbc.Dockerfile \
-t ${IMAGE_NAME} \
--build-arg LINKIS_IMAGE=${LINKIS_IMAGE} \
diff --git a/linkis-dist/docker/scripts/prepare-ldh-image.sh b/linkis-dist/docker/scripts/prepare-ldh-image.sh
index db846b705..44623a426 100755
--- a/linkis-dist/docker/scripts/prepare-ldh-image.sh
+++ b/linkis-dist/docker/scripts/prepare-ldh-image.sh
@@ -34,6 +34,7 @@ SPARK_VERSION=${SPARK_VERSION:-2.4.3}
SPARK_HADOOP_VERSION=${SPARK_HADOOP_VERSION:-2.7}
FLINK_VERSION=${FLINK_VERSION:-1.12.2}
ZOOKEEPER_VERSION=${ZOOKEEPER_VERSION:-3.5.9}
+MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION:-8.0.28}
set -e
@@ -44,6 +45,7 @@ TARFILENAME_HIVE="apache-hive-${HIVE_VERSION}-bin.tar.gz"
TARFILENAME_SPARK="spark-${SPARK_VERSION}-bin-hadoop${SPARK_HADOOP_VERSION}.tgz"
TARFILENAME_FLINK="flink-${FLINK_VERSION}-bin-scala_2.11.tgz"
TARFILENAME_ZOOKEEPER="apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz"
+TARFILENAME_MYSQL_JDBC=mysql-connector-java-${MYSQL_JDBC_VERSION}.jar
DOWNLOAD_URL_HADOOP="https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/${TARFILENAME_HADOOP}"
DOWNLOAD_URL_HIVE="https://archive.apache.org/dist/hive/hive-${HIVE_VERSION}/${TARFILENAME_HIVE}"
@@ -51,8 +53,15 @@ DOWNLOAD_URL_SPARK="https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}
DOWNLOAD_URL_FLINK="https://archive.apache.org/dist/flink/flink-${FLINK_VERSION}/${TARFILENAME_FLINK}"
DOWNLOAD_URL_ZOOKEEPER="https://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/${TARFILENAME_ZOOKEEPER}"
+
+
+DOWNLOAD_URL_MYSQL_JDBC="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_JDBC_VERSION}/${TARFILENAME_MYSQL_JDBC}"
+
+
+
download ${DOWNLOAD_URL_HADOOP} ${TARFILENAME_HADOOP} ${LDH_TAR_DIR}
download ${DOWNLOAD_URL_HIVE} ${TARFILENAME_HIVE} ${LDH_TAR_DIR}
download ${DOWNLOAD_URL_SPARK} ${TARFILENAME_SPARK} ${LDH_TAR_DIR}
download ${DOWNLOAD_URL_FLINK} ${TARFILENAME_FLINK} ${LDH_TAR_DIR}
download ${DOWNLOAD_URL_ZOOKEEPER} ${TARFILENAME_ZOOKEEPER} ${LDH_TAR_DIR}
+download ${DOWNLOAD_URL_MYSQL_JDBC} ${TARFILENAME_MYSQL_JDBC} ${LDH_TAR_DIR}
\ No newline at end of file
diff --git a/linkis-dist/docker/scripts/utils.sh b/linkis-dist/docker/scripts/utils.sh
index bc0e579e3..f7813cfe7 100755
--- a/linkis-dist/docker/scripts/utils.sh
+++ b/linkis-dist/docker/scripts/utils.sh
@@ -27,9 +27,10 @@ download() {
echo "- ${TAR_FILE} already exists in ${TAR_CACHE_ROOT}, downloading skipped."
fi
- echo "- create hard link: ${HARD_LINK_ROOT}/${TAR_FILE} -> ${TAR_CACHE_ROOT}/${TAR_FILE}"
+ echo "- cp: ${TAR_CACHE_ROOT}/${TAR_FILE} -> ${HARD_LINK_ROOT}/${TAR_FILE} "
rm -rf ${HARD_LINK_ROOT}/${TAR_FILE}
- ln ${TAR_CACHE_ROOT}/${TAR_FILE} ${HARD_LINK_ROOT}/${TAR_FILE}
+ # ln maybe cause invalid cross-device link
+ cp ${TAR_CACHE_ROOT}/${TAR_FILE} ${HARD_LINK_ROOT}/${TAR_FILE}
}
WORK_DIR=`cd $(dirname $0); pwd -P`
diff --git a/linkis-dist/helm/README.md b/linkis-dist/helm/README.md
index b27d5a371..cbe2f5037 100644
--- a/linkis-dist/helm/README.md
+++ b/linkis-dist/helm/README.md
@@ -151,7 +151,7 @@ $> sh ./scripts/login-pod.sh mg-gateway
```
```shell
# in the mg-gateway container
-bash-4.2$ ./bin/./linkis-cli -engineType shell-1 -codeType shell -code "echo \"hello\" " -submitUser hadoop -proxyUser hadoop
+bash-4.2$ ./bin/linkis-cli -engineType shell-1 -codeType shell -code "echo \"hello\" " -submitUser hadoop -proxyUser hadoop
=====Java Start Command=====
exec /etc/alternatives/jre/bin/java -server -Xms32m -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/linkis/logs/linkis-cli -XX:ErrorFile=/opt/linkis/logs/linkis-cli/ps_err_pid%p.log -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=80 -XX:+DisableExplicitGC -classpath /opt/linkis/conf/linkis-cli:/opt/linkis/lib/linkis-computation-governance/linkis-client/linkis-cli/*:/opt/linkis/lib/linkis-commons/public-module/*: -Dconf.root=/etc/linkis-conf -Dconf.file=linki [...]
diff --git a/linkis-dist/helm/README_CN.md b/linkis-dist/helm/README_CN.md
index 06b3e08e9..83c843767 100644
--- a/linkis-dist/helm/README_CN.md
+++ b/linkis-dist/helm/README_CN.md
@@ -149,7 +149,7 @@ $> sh ./scripts/login-pod.sh mg-gateway
```
```shell
# mg-gateway 容器内
-bash-4.2$ ./bin/./linkis-cli -engineType shell-1 -codeType shell -code "echo \"hello\" " -submitUser hadoop -proxyUser hadoop
+bash-4.2$ ./bin/linkis-cli -engineType shell-1 -codeType shell -code "echo \"hello\" " -submitUser hadoop -proxyUser hadoop
=====Java Start Command=====
exec /etc/alternatives/jre/bin/java -server -Xms32m -Xmx2048m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/opt/linkis/logs/linkis-cli -XX:ErrorFile=/opt/linkis/logs/linkis-cli/ps_err_pid%p.log -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=80 -XX:+DisableExplicitGC -classpath /opt/linkis/conf/linkis-cli:/opt/linkis/lib/linkis-computation-governance/linkis-client/linkis-cli/*:/opt/linkis/lib/linkis-commons/public-module/*: -Dconf.root=/etc/linkis-conf -Dconf.file=linki [...]
diff --git a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
index cace29327..6a9d91e2e 100644
--- a/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
+++ b/linkis-dist/helm/charts/linkis/templates/linkis-cg-engineconnmanager.yaml
@@ -180,6 +180,10 @@ spec:
{{- if eq .Values.linkis.featureGates.localMode false }}
- name: hadoop-conf
mountPath: /etc/hadoop-conf
+ - name: hive-conf
+ mountPath: /etc/hive-conf
+ - name: spark-conf
+ mountPath: /etc/spark-conf
{{- end }}
- name: conf
mountPath: {{ .Values.linkis.locations.confDir }}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org