You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2022/09/17 13:50:11 UTC
[incubator-linkis] branch dev-1.3.0 updated: [Optimization] speed up image creation with base image (#3441)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.3.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.0 by this push:
new 2ea65a78c [Optimization] speed up image creation with base image (#3441)
2ea65a78c is described below
commit 2ea65a78cb979e55f71206729280592cdab5951f
Author: Aaron Lin <51...@users.noreply.github.com>
AuthorDate: Sat Sep 17 21:50:06 2022 +0800
[Optimization] speed up image creation with base image (#3441)
* [Optimization] speed up image creation with base image
---
.github/workflows/publish-docker.yaml | 2 +-
README.md | 26 ++++++++++++-
linkis-dist/docker/ldh.Dockerfile | 2 +-
.../docker/linkis-with-mysql-jdbc.Dockerfile | 29 ++++++++++++++
linkis-dist/docker/linkis.Dockerfile | 42 +++++++++------------
.../scripts/make-linikis-image-with-mysql-jdbc.sh | 44 ++++++++++++++++++++++
linkis-dist/docker/scripts/prepare-ldh-image.sh | 22 +----------
linkis-dist/docker/scripts/utils.sh | 39 +++++++++++++++++++
linkis-dist/helm/README.md | 5 ++-
linkis-dist/pom.xml | 31 +++++++++++----
10 files changed, 185 insertions(+), 57 deletions(-)
diff --git a/.github/workflows/publish-docker.yaml b/.github/workflows/publish-docker.yaml
index c1701f481..8ef2de273 100644
--- a/.github/workflows/publish-docker.yaml
+++ b/.github/workflows/publish-docker.yaml
@@ -66,7 +66,7 @@ jobs:
./mvnw -N install
- name: Build backend by maven
run: |
- ./mvnw install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true -Dimage.build.type=release
+ ./mvnw install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true
- name: Log in to the Container registry
uses: docker/login-action@v1.10.0
with:
diff --git a/README.md b/README.md
index 464d34dd4..7786edf90 100644
--- a/README.md
+++ b/README.md
@@ -140,7 +140,10 @@ Please go to the [Linkis Releases Page](https://linkis.apache.org/download/main)
# - Option 1: make the linkis distribution package only
./mvnw clean install -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
# - Option 2: make the linkis distribution package and docker image
+# - Option 2.1: image without mysql jdbc jars
./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true
+# - Option 2.2: image with mysql jdbc jars
+./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.with.jdbc=true
# - Option 3: linkis distribution package and docker image (included web)
./mvnw clean install -Pdocker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -Dlinkis.build.web=true
# - Option 4: linkis distribution package and docker image (included web and ldh (hadoop all in one for test))
@@ -156,7 +159,28 @@ cd incubator-linkis/linkis-web
npm install
npm run build
```
-
+
+### Bundled with MySQL JDBC Driver
+Due to the MySQL licensing restrictions, the MySQL Java Database Connectivity (JDBC) driver is not bundled with the
+official released linkis image by default. However, at current stage, linkis still relies on this library to work properly.
+To solve this problem, we provide a script which can help to creating an custom image with mysql jdbc from the official
+linkis image by yourself, the image created by this tool will be tagged as `linkis:with-jdbc` by default.
+
+```shell
+$> LINKIS_IMAGE=linkis:1.3.0 ./linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
+
+# build dir: ...
+# base image: linkis:1.3.0
+# mysql jdbc version: 8.0.28
+... 0.0s
+ => exporting to image 0.0s
+ => => exporting layers 0.0s
+ => => writing image sha256:3870df5500a71fcf879b5b7d5699c3c9804c7e03e33ad842e5d11f3504371fe8 0.0s
+ => => naming to docker.io/library/linkis:with-jdbc 0.0s
+# done, image: linkis:with-jdbc
+```
+
+
Please refer to [Quick Deployment](https://linkis.apache.org/docs/latest/deployment/quick-deploy) to do the deployment.
# Examples and Guidance
diff --git a/linkis-dist/docker/ldh.Dockerfile b/linkis-dist/docker/ldh.Dockerfile
index 87f164404..dc6b9ee2b 100644
--- a/linkis-dist/docker/ldh.Dockerfile
+++ b/linkis-dist/docker/ldh.Dockerfile
@@ -25,7 +25,7 @@ FROM ${IMAGE_BASE} as linkis-ldh
ARG JDK_VERSION=1.8.0-openjdk
ARG JDK_BUILD_REVISION=1.8.0.332.b09-1.el7_9
-ARG MYSQL_JDBC_VERSION=5.1.49
+ARG MYSQL_JDBC_VERSION=8.0.28
ARG HADOOP_VERSION=2.7.2
ARG HIVE_VERSION=2.3.3
diff --git a/linkis-dist/docker/linkis-with-mysql-jdbc.Dockerfile b/linkis-dist/docker/linkis-with-mysql-jdbc.Dockerfile
new file mode 100644
index 000000000..e8fed01d7
--- /dev/null
+++ b/linkis-dist/docker/linkis-with-mysql-jdbc.Dockerfile
@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ARG LINKIS_IMAGE=linkis:dev
+
+######################################################################
+# linkis image with mysql jdbc
+######################################################################
+FROM ${LINKIS_IMAGE}
+
+ARG LINKIS_HOME=/opt/linkis
+ARG MYSQL_JDBC_VERSION=8.0.28
+
+COPY mysql-connector-java-${MYSQL_JDBC_VERSION}.jar ${LINKIS_HOME}/lib/linkis-commons/public-module/
+COPY mysql-connector-java-${MYSQL_JDBC_VERSION}.jar ${LINKIS_HOME}/lib/linkis-spring-cloud-services/linkis-mg-gateway/
diff --git a/linkis-dist/docker/linkis.Dockerfile b/linkis-dist/docker/linkis.Dockerfile
index ba7657206..62162ed18 100644
--- a/linkis-dist/docker/linkis.Dockerfile
+++ b/linkis-dist/docker/linkis.Dockerfile
@@ -15,21 +15,30 @@
# limitations under the License.
#
-######################################################################
-# linkis image
-######################################################################
-
ARG IMAGE_BASE=centos:7
ARG IMAGE_BASE_WEB=nginx:1.19.6
-FROM ${IMAGE_BASE} as linkis
-
-ARG BUILD_TYPE=dev
+######################################################################
+# linkis image base
+######################################################################
+FROM ${IMAGE_BASE} as linkis-base
ARG JDK_VERSION=1.8.0-openjdk
ARG JDK_BUILD_REVISION=1.8.0.332.b09-1.el7_9
-ARG MYSQL_JDBC_VERSION=5.1.49
+# TODO: remove install mysql client when schema-init-tools is ready
+RUN yum install -y \
+ vim unzip curl sudo krb5-workstation sssd crontabs python-pip \
+ java-${JDK_VERSION}-${JDK_BUILD_REVISION} \
+ java-${JDK_VERSION}-devel-${JDK_BUILD_REVISION} \
+ mysql \
+ && yum clean all
+
+
+######################################################################
+# linkis image for release
+######################################################################
+FROM linkis-base as linkis
ARG LINKIS_VERSION=0.0.0
ARG LINKIS_SYSTEM_USER="hadoop"
@@ -42,15 +51,6 @@ ARG LINKIS_LOG_DIR=/var/logs/linkis
WORKDIR ${LINKIS_HOME}
RUN useradd -r -s /bin/bash -u ${LINKIS_SYSTEM_UID} -g root -G wheel ${LINKIS_SYSTEM_USER}
-
-# TODO: remove install mysql client when schema-init-tools is ready
-RUN yum install -y \
- vim unzip curl sudo krb5-workstation sssd crontabs python-pip \
- java-${JDK_VERSION}-${JDK_BUILD_REVISION} \
- java-${JDK_VERSION}-devel-${JDK_BUILD_REVISION} \
- mysql \
- && yum clean all
-
RUN sed -i "s#^%wheel.*#%wheel ALL=(ALL) NOPASSWD: ALL#g" /etc/sudoers
RUN mkdir -p /opt/tmp \
@@ -67,13 +67,6 @@ ADD apache-linkis-${LINKIS_VERSION}-incubating-bin.tar.gz /opt/tmp/
RUN mv /opt/tmp/linkis-package/* ${LINKIS_HOME}/ \
&& rm -rf /opt/tmp
-# Put mysql-connector-java-*.jar package into the image only in development mode
-RUN if [ "$BUILD_TYPE" = "dev" ] ; then \
- curl -L -o ${LINKIS_HOME}/lib/linkis-commons/public-module/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar \
- https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_JDBC_VERSION}/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar \
- && cp ${LINKIS_HOME}/lib/linkis-commons/public-module/mysql-connector-java-${MYSQL_JDBC_VERSION}.jar ${LINKIS_HOME}/lib/linkis-spring-cloud-services/linkis-mg-gateway/ ;\
- fi
-
RUN chmod g+w -R ${LINKIS_HOME} && chown ${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LINKIS_HOME} \
&& chmod g+w -R ${LINKIS_CONF_DIR} && chown ${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LINKIS_CONF_DIR} \
&& chmod g+w -R ${LINKIS_LOG_DIR} && chown ${LINKIS_SYSTEM_USER}:${LINKIS_SYSTEM_GROUP} -R ${LINKIS_LOG_DIR} \
@@ -84,7 +77,6 @@ USER ${LINKIS_SYSTEM_USER}
ENTRYPOINT ["/bin/bash"]
-
######################################################################
# linkis web image
######################################################################
diff --git a/linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh b/linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
new file mode 100755
index 000000000..1b722f41a
--- /dev/null
+++ b/linkis-dist/docker/scripts/make-linikis-image-with-mysql-jdbc.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+WORK_DIR=`cd $(dirname $0); pwd -P`
+
+. ${WORK_DIR}/utils.sh
+
+IMAGE_NAME=${IMAGE_NAME:-linkis:with-jdbc}
+LINKIS_IMAGE=${LINKIS_IMAGE:-linkis:dev}
+LINKIS_HOME=${LINKIS_HOME:-/opt/linkis}
+MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION:-8.0.28}
+MYSQL_JDBC_FILENAME=mysql-connector-java-${MYSQL_JDBC_VERSION}.jar
+MYSQL_JDBC_URL="https://repo1.maven.org/maven2/mysql/mysql-connector-java/${MYSQL_JDBC_VERSION}/${MYSQL_JDBC_FILENAME}"
+
+BUILD_DIR=`mktemp -d -t linkis-build-XXXXX`
+
+echo "# build dir: ${BUILD_DIR}"
+echo "# base image: ${LINKIS_IMAGE}"
+echo "# mysql jdbc version: ${MYSQL_JDBC_VERSION}"
+
+download ${MYSQL_JDBC_URL} ${MYSQL_JDBC_FILENAME} ${BUILD_DIR}
+
+docker build -f ${WORK_DIR}/../linkis-with-mysql-jdbc.Dockerfile \
+ -t ${IMAGE_NAME} \
+ --build-arg LINKIS_IMAGE=${LINKIS_IMAGE} \
+ --build-arg LINKIS_HOME=${LINKIS_HOME} \
+ --build-arg MYSQL_JDBC_VERSION=${MYSQL_JDBC_VERSION} \
+ ${BUILD_DIR}
+
+echo "# done, image: ${IMAGE_NAME}"
diff --git a/linkis-dist/docker/scripts/prepare-ldh-image.sh b/linkis-dist/docker/scripts/prepare-ldh-image.sh
index dce3f8fde..db846b705 100755
--- a/linkis-dist/docker/scripts/prepare-ldh-image.sh
+++ b/linkis-dist/docker/scripts/prepare-ldh-image.sh
@@ -15,28 +15,10 @@
#
#
-download() {
- TAR_URL=$1
- TAR_FILE=$2
- HARD_LINK_ROOT=$3
-
- if [ ! -f ${TAR_CACHE_ROOT}/${TAR_FILE} ]; then
- echo "- downloading ${TAR_FILE} to ${TAR_CACHE_ROOT} from ${TAR_URL}"
- curl -L ${TAR_URL} -o ${TAR_CACHE_ROOT}/${TAR_FILE}
- else
- echo "- ${TAR_FILE} already exists in ${TAR_CACHE_ROOT}, downloading skipped."
- fi
-
- echo "- create hard link: ${HARD_LINK_ROOT}/${TAR_FILE} -> ${TAR_CACHE_ROOT}/${TAR_FILE}"
- rm -rf ${HARD_LINK_ROOT}/${TAR_FILE}
- ln ${TAR_CACHE_ROOT}/${TAR_FILE} ${HARD_LINK_ROOT}/${TAR_FILE}
-}
-
WORK_DIR=`cd $(dirname $0); pwd -P`
-PROJECT_ROOT=${WORK_DIR}/../..
-PROJECT_TARGET=${PROJECT_ROOT}/target
-TAR_CACHE_ROOT=${HOME}/.linkis-build-cache
+. ${WORK_DIR}/utils.sh
+
LDH_TAR_DIR=${PROJECT_TARGET}/ldh-tars
mkdir -p ${TAR_CACHE_ROOT}
diff --git a/linkis-dist/docker/scripts/utils.sh b/linkis-dist/docker/scripts/utils.sh
new file mode 100755
index 000000000..bc0e579e3
--- /dev/null
+++ b/linkis-dist/docker/scripts/utils.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+
+download() {
+ TAR_URL=$1
+ TAR_FILE=$2
+ HARD_LINK_ROOT=$3
+
+ if [ ! -f ${TAR_CACHE_ROOT}/${TAR_FILE} ]; then
+ echo "- downloading ${TAR_FILE} to ${TAR_CACHE_ROOT} from ${TAR_URL}"
+ curl -L ${TAR_URL} -o ${TAR_CACHE_ROOT}/${TAR_FILE}
+ else
+ echo "- ${TAR_FILE} already exists in ${TAR_CACHE_ROOT}, downloading skipped."
+ fi
+
+ echo "- create hard link: ${HARD_LINK_ROOT}/${TAR_FILE} -> ${TAR_CACHE_ROOT}/${TAR_FILE}"
+ rm -rf ${HARD_LINK_ROOT}/${TAR_FILE}
+ ln ${TAR_CACHE_ROOT}/${TAR_FILE} ${HARD_LINK_ROOT}/${TAR_FILE}
+}
+
+WORK_DIR=`cd $(dirname $0); pwd -P`
+
+PROJECT_ROOT=${WORK_DIR}/../..
+PROJECT_TARGET=${PROJECT_ROOT}/target
+TAR_CACHE_ROOT=${HOME}/.linkis-build-cache
diff --git a/linkis-dist/helm/README.md b/linkis-dist/helm/README.md
index 22818decd..efd38294f 100644
--- a/linkis-dist/helm/README.md
+++ b/linkis-dist/helm/README.md
@@ -207,7 +207,8 @@ $> ./mvnw clean install -Pdocker \
-Dmaven.javadoc.skip=true \
-Dmaven.test.skip=true \
-Dlinkis.build.web=true \
- -Dlinkis.build.ldh=true
+ -Dlinkis.build.ldh=true \
+ -Dlinkis.build.with.jdbc=true
```
By default, we download the pre-built binary distributions for each hadoop component from the official site of [Apache Archives](https://archive.apache.org/dist/), which can be very slow for members in some regions.
@@ -220,7 +221,7 @@ Run the following command to setup a local kubernetes cluster with LDH on it.
$> sh ./scripts/create-kind-cluster.sh \
&& sh ./scripts/install-mysql.sh \
&& sh ./scripts/install-ldh.sh \
- && sh ./scripts/install-charts.sh
+ && sh ./scripts/install-charts-with-ldh.sh
...
diff --git a/linkis-dist/pom.xml b/linkis-dist/pom.xml
index 63e19c9f3..835b73f50 100644
--- a/linkis-dist/pom.xml
+++ b/linkis-dist/pom.xml
@@ -202,7 +202,6 @@
<profile>
<id>docker</id>
<properties>
- <image.build.type>dev</image.build.type>
<image.base>centos:7</image.base>
<image.base.web>nginx:1.19.6</image.base.web>
<jdk.version>1.8.0-openjdk</jdk.version>
@@ -250,10 +249,6 @@
<arg value="--build-arg"></arg>
<arg value="LINKIS_VERSION=${project.version}"></arg>
<arg value="--build-arg"></arg>
- <arg value="MYSQL_JDBC_VERSION=${mysql.connector.version}"></arg>
- <arg value="--build-arg"></arg>
- <arg value="BUILD_TYPE=${image.build.type}"></arg>
- <arg value="--build-arg"></arg>
<arg value="LINKIS_SYSTEM_USER=${linkis.system.user}"></arg>
<arg value="--build-arg"></arg>
<arg value="LINKIS_SYSTEM_UID=${linkis.system.uid}"></arg>
@@ -273,6 +268,30 @@
</target>
</configuration>
</execution>
+ <execution>
+ <id>build-linkis-image-with-mysql-jdbc</id>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <phase>install</phase>
+ <configuration>
+ <target if="linkis.build.with.jdbc" name="linkis-image-with-jdbc">
+ <echo message="Building linkis image with mysql jdbc ..."></echo>
+ <exec executable="sh" failonerror="true">
+ <env key="IMAGE_NAME" value="${project.parent.artifactId}:${project.version}-with-jdbc"></env>
+ <env key="LINKIS_IMAGE" value="${project.parent.artifactId}:${project.version}"></env>
+ <env key="LINKIS_HOME" value="${linkis.home}"></env>
+ <env key="MYSQL_JDBC_VERSION" value="${mysql.connector.version}"></env>
+ <arg value="${basedir}/docker/scripts/make-linikis-image-with-mysql-jdbc.sh"></arg>
+ </exec>
+ <exec executable="docker" failonerror="true">
+ <arg value="tag"></arg>
+ <arg value="${project.parent.artifactId}:${project.version}-with-jdbc"></arg>
+ <arg value="${project.parent.artifactId}:dev"></arg>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
<execution>
<id>build-linkis-web-image</id>
<goals>
@@ -347,8 +366,6 @@
<arg value="--build-arg"></arg>
<arg value="MYSQL_JDBC_VERSION=${mysql.connector.version}"></arg>
<arg value="--build-arg"></arg>
- <arg value="BUILD_TYPE=${image.build.type}"></arg>
- <arg value="--build-arg"></arg>
<arg value="LINKIS_SYSTEM_USER=${linkis.system.user}"></arg>
<arg value="--build-arg"></arg>
<arg value="LINKIS_SYSTEM_UID=${linkis.system.uid}"></arg>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org