You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@druid.apache.org by ab...@apache.org on 2022/04/11 14:58:21 UTC
[druid] branch master updated: Fix zulu8 set-up Dockerfile for hadoop and hadoop3 in hadoop ingestion tutorial (#12248)
This is an automated email from the ASF dual-hosted git repository.
abhishek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
The following commit(s) were added to refs/heads/master by this push:
new 665c926824 Fix zulu8 set-up Dockerfile for hadoop and hadoop3 in hadoop ingestion tutorial (#12248)
665c926824 is described below
commit 665c926824375fd096db805e7140d4be1d361dc1
Author: Tiffany Yeh <33...@users.noreply.github.com>
AuthorDate: Mon Apr 11 10:58:09 2022 -0400
Fix zulu8 set-up Dockerfile for hadoop and hadoop3 in hadoop ingestion tutorial (#12248)
Fix errors related to zulu8 installation for building the Hadoop Docker image in the Load From Apache Hadoop tutorial.
The steps to download zulu8 in the Dockerfile and setup-zulu-repo.sh were replaced with the steps in the Dockerfile released by zulu-openjdk: https://github.com/zulu-openjdk/zulu-openjdk/blob/be45d20302e42df5aa95d2de078bb5e4214f5dba/centos/8u282-8.52.0.23/Dockerfile.
---
.../quickstart/tutorial/hadoop/docker/Dockerfile | 22 +++++--
.../tutorial/hadoop/docker/setup-zulu-repo.sh | 67 ----------------------
.../quickstart/tutorial/hadoop3/docker/Dockerfile | 22 +++++--
.../tutorial/hadoop3/docker/setup-zulu-repo.sh | 67 ----------------------
4 files changed, 32 insertions(+), 146 deletions(-)
diff --git a/examples/quickstart/tutorial/hadoop/docker/Dockerfile b/examples/quickstart/tutorial/hadoop/docker/Dockerfile
index 0237d61cbb..fd71f5942f 100644
--- a/examples/quickstart/tutorial/hadoop/docker/Dockerfile
+++ b/examples/quickstart/tutorial/hadoop/docker/Dockerfile
@@ -36,12 +36,22 @@ RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa
RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-# zulu java 8
-COPY setup-zulu-repo.sh /root/setup-zulu-repo.sh
-RUN /root/setup-zulu-repo.sh
-RUN yum install -y zulu-8
+#
+# Pull Zulu OpenJDK binaries from official repository:
+#
+
+ARG ZULU_REPO_VER=1.0.0-1
+
+RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \
+ curl -sLO https://cdn.azul.com/zulu/bin/zulu-repo-${ZULU_REPO_VER}.noarch.rpm && \
+ rpm -ivh zulu-repo-${ZULU_REPO_VER}.noarch.rpm && \
+ yum -q -y update && \
+ yum -q -y upgrade && \
+ yum -q -y install zulu8-jdk && \
+ yum clean all && \
+ rm -rf /var/cache/yum zulu-repo_${ZULU_REPO_VER}.noarch.rpm
-ENV JAVA_HOME /usr/lib/jvm/zulu-8
+ENV JAVA_HOME=/usr/lib/jvm/zulu8
ENV PATH $PATH:$JAVA_HOME/bin
# hadoop
@@ -57,7 +67,7 @@ ENV HADOOP_YARN_HOME /usr/local/hadoop
ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
ENV YARN_CONF_DIR $HADOOP_PREFIX/etc/hadoop
-RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/zulu-8\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
+RUN sed -i '/^export JAVA_HOME/ s:.*:export JAVA_HOME=/usr/lib/jvm/zulu8\nexport HADOOP_PREFIX=/usr/local/hadoop\nexport HADOOP_HOME=/usr/local/hadoop\n:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
RUN sed -i '/^export HADOOP_CONF_DIR/ s:.*:export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/:' $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
RUN mkdir $HADOOP_PREFIX/input
diff --git a/examples/quickstart/tutorial/hadoop/docker/setup-zulu-repo.sh b/examples/quickstart/tutorial/hadoop/docker/setup-zulu-repo.sh
deleted file mode 100755
index 087240d2a0..0000000000
--- a/examples/quickstart/tutorial/hadoop/docker/setup-zulu-repo.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Script to set up the Azul Zulu JDK yum repository.
-#
-
-# Hardcode GPG key so we don't have to fetch it over http.
-cat <<'EOT' > /root/RPM-GPG-KEY-azulsystems
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mQINBFNgFa8BEADTL/REB10M+TfiZOtFHqL5LHKkzTMn/O2r5iIqXGhi6iwZazFs
-9S5g1eU7WMen5Xp9AREs+OvaHx91onPZ7ZiP7VpZ6ZdwWrnVk1Y/HfI59tWxmNYW
-DmKYBGMj4EUpFPSE9EnFj7dm1WdlCvpognCwZQl9D3BseGqN7OLHfwqqmOlbYN9h
-HYkT+CaqOoWDIGMB3UkBlMr0GuujEP8N1gxg7EOcSCsZH5aKtXubdUlVSphfAAwD
-z4MviB39J22sPBnKmaOT3TUTO5vGeKtC9BAvtgA82jY2TtCEjetnfK/qtzj/6j2N
-xVUbHQydwNQVRU92A7334YvCbn3xUUNI0WOscdmfpgCU0Z9Gb2IqDb9cMjgUi8F6
-MG/QY9/CZjX62XrHRPm3aXsCJOVh/PO1sl2A/rvv8AkpJKYyhm6T8OBFptCsA3V4
-Oic7ZyYhqV0u2r4NON+1MoUeuuoeY2tIrbRxe3ffVOxPzrESzSbc8LC2tYaP+wGd
-W0f57/CoDkUzlvpReCUI1Bv5zP4/jhC63Rh6lffvSf2tQLwOsf5ivPhUtwUfOQjg
-v9P8Wc8K7XZpSOMnDZuDe9wuvB/DiH/P5yiTs2RGsbDdRh5iPfwbtf2+IX6h2lNZ
-XiDKt9Gc26uzeJRx/c7+sLunxq6DLIYvrsEipVI9frHIHV6fFTmqMJY6SwARAQAB
-tEdBenVsIFN5c3RlbXMsIEluYy4gKFBhY2thZ2Ugc2lnbmluZyBrZXkuKSA8cGtp
-LXNpZ25pbmdAYXp1bHN5c3RlbXMuY29tPokCOAQTAQIAIgUCU2AVrwIbAwYLCQgH
-AwIGFQgCCQoLBBYCAwECHgECF4AACgkQsZmDYSGb2cnJ8xAAz1V1PJnfOyaRIP2N
-Ho2uRwGdPsA4eFMXb4Z08eGjDMD3b9WW3D0XnCLbJpaZ6klz0W0s2tcYSneTBaSs
-RAqxgJgBZ5ZMXtrrHld/5qFoBbStLZLefmcPhnfvamwHDCTLUex8NIAI1u3e9Rhb
-5fbH+gpuYpwHX7hz0FOfpn1sxR03UyxU+ey4AdKe9LG3TJVnB0WcgxpobpbqweLH
-yzcEQCNoFV3r1rlE13Y0aE31/9apoEwiYvqAzEmE38TukDLl/Qg8rkR1t0/lok2P
-G6pWqdN7pmoUovBTvDi5YOthcjZcdOTXXn2Yw4RZVF9uhRsVfku1Eg25SnOje3uY
-smtQLME4eESbePdjyV/okCIle66uHZse+7gNyNmWpf01hM+VmAySIAyKa0Ku8AXZ
-MydEcJTebrNfW9uMLsBx3Ts7z/CBfRng6F8louJGlZtlSwddTkZVcb26T20xeo0a
-ZvdFXM2djTi/a5nbBoZQL85AEeV7HaphFLdPrgmMtS8sSZUEVvdaxp7WJsVuF9cO
-Nxsvx40OYTvfco0W41Lm8/sEuQ7YueEVpZxiv5kX56GTU9vXaOOi+8Z7Ee2w6Adz
-4hrGZkzztggs4tM9geNYnd0XCdZ/ICAskKJABg7biDD1PhEBrqCIqSE3U497vibQ
-Mpkkl/Zpp0BirhGWNyTg8K4JrsQ=
-=d320
------END PGP PUBLIC KEY BLOCK-----
-EOT
-
-rpm --import /root/RPM-GPG-KEY-azulsystems
-rpm --rebuilddb
-
-# Do not include "gpgkey" in the repo definition -- we've already imported it, above.
-cat <<'EOT' > /etc/yum.repos.d/zulu.repo
-[zulu]
-name=zulu-$releasever - Azul Systems Inc., Zulu packages for $basearch
-baseurl=http://repos.azulsystems.com/rhel/$releasever/$basearch
-enabled=1
-gpgcheck=1
-protect=1
-EOT
diff --git a/examples/quickstart/tutorial/hadoop3/docker/Dockerfile b/examples/quickstart/tutorial/hadoop3/docker/Dockerfile
index 0e23f24e40..c0475651ee 100644
--- a/examples/quickstart/tutorial/hadoop3/docker/Dockerfile
+++ b/examples/quickstart/tutorial/hadoop3/docker/Dockerfile
@@ -36,12 +36,22 @@ RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
RUN ssh-keygen -q -N "" -t rsa -f /root/.ssh/id_rsa
RUN cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys
-# zulu java 8
-COPY setup-zulu-repo.sh /root/setup-zulu-repo.sh
-RUN /root/setup-zulu-repo.sh
-RUN yum install -y zulu-8
+#
+# Pull Zulu OpenJDK binaries from official repository:
+#
+
+ARG ZULU_REPO_VER=1.0.0-1
+
+RUN rpm --import http://repos.azulsystems.com/RPM-GPG-KEY-azulsystems && \
+ curl -sLO https://cdn.azul.com/zulu/bin/zulu-repo-${ZULU_REPO_VER}.noarch.rpm && \
+ rpm -ivh zulu-repo-${ZULU_REPO_VER}.noarch.rpm && \
+ yum -q -y update && \
+ yum -q -y upgrade && \
+ yum -q -y install zulu8-jdk && \
+ yum clean all && \
+ rm -rf /var/cache/yum zulu-repo_${ZULU_REPO_VER}.noarch.rpm
-ENV JAVA_HOME /usr/lib/jvm/zulu-8
+ENV JAVA_HOME=/usr/lib/jvm/zulu8
ENV PATH $PATH:$JAVA_HOME/bin
# hadoop
@@ -58,7 +68,7 @@ ENV HADOOP_CONF_DIR /usr/local/hadoop/etc/hadoop
ENV YARN_CONF_DIR $HADOOP_HOME/etc/hadoop
# in hadoop 3 the example file is nearly empty so we can just append stuff
-RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu-8' $HADOOP_HOME/etc/hadoop/hadoop-env.sh
+RUN sed -i '$ a export JAVA_HOME=/usr/lib/jvm/zulu8' $HADOOP_HOME/etc/hadoop/hadoop-env.sh
RUN sed -i '$ a export HADOOP_HOME=/usr/local/hadoop' $HADOOP_HOME/etc/hadoop/hadoop-env.sh
RUN sed -i '$ a export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/' $HADOOP_HOME/etc/hadoop/hadoop-env.sh
RUN sed -i '$ a export HDFS_NAMENODE_USER=root' $HADOOP_HOME/etc/hadoop/hadoop-env.sh
diff --git a/examples/quickstart/tutorial/hadoop3/docker/setup-zulu-repo.sh b/examples/quickstart/tutorial/hadoop3/docker/setup-zulu-repo.sh
deleted file mode 100755
index 087240d2a0..0000000000
--- a/examples/quickstart/tutorial/hadoop3/docker/setup-zulu-repo.sh
+++ /dev/null
@@ -1,67 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-#
-# Script to set up the Azul Zulu JDK yum repository.
-#
-
-# Hardcode GPG key so we don't have to fetch it over http.
-cat <<'EOT' > /root/RPM-GPG-KEY-azulsystems
------BEGIN PGP PUBLIC KEY BLOCK-----
-Version: GnuPG v1
-
-mQINBFNgFa8BEADTL/REB10M+TfiZOtFHqL5LHKkzTMn/O2r5iIqXGhi6iwZazFs
-9S5g1eU7WMen5Xp9AREs+OvaHx91onPZ7ZiP7VpZ6ZdwWrnVk1Y/HfI59tWxmNYW
-DmKYBGMj4EUpFPSE9EnFj7dm1WdlCvpognCwZQl9D3BseGqN7OLHfwqqmOlbYN9h
-HYkT+CaqOoWDIGMB3UkBlMr0GuujEP8N1gxg7EOcSCsZH5aKtXubdUlVSphfAAwD
-z4MviB39J22sPBnKmaOT3TUTO5vGeKtC9BAvtgA82jY2TtCEjetnfK/qtzj/6j2N
-xVUbHQydwNQVRU92A7334YvCbn3xUUNI0WOscdmfpgCU0Z9Gb2IqDb9cMjgUi8F6
-MG/QY9/CZjX62XrHRPm3aXsCJOVh/PO1sl2A/rvv8AkpJKYyhm6T8OBFptCsA3V4
-Oic7ZyYhqV0u2r4NON+1MoUeuuoeY2tIrbRxe3ffVOxPzrESzSbc8LC2tYaP+wGd
-W0f57/CoDkUzlvpReCUI1Bv5zP4/jhC63Rh6lffvSf2tQLwOsf5ivPhUtwUfOQjg
-v9P8Wc8K7XZpSOMnDZuDe9wuvB/DiH/P5yiTs2RGsbDdRh5iPfwbtf2+IX6h2lNZ
-XiDKt9Gc26uzeJRx/c7+sLunxq6DLIYvrsEipVI9frHIHV6fFTmqMJY6SwARAQAB
-tEdBenVsIFN5c3RlbXMsIEluYy4gKFBhY2thZ2Ugc2lnbmluZyBrZXkuKSA8cGtp
-LXNpZ25pbmdAYXp1bHN5c3RlbXMuY29tPokCOAQTAQIAIgUCU2AVrwIbAwYLCQgH
-AwIGFQgCCQoLBBYCAwECHgECF4AACgkQsZmDYSGb2cnJ8xAAz1V1PJnfOyaRIP2N
-Ho2uRwGdPsA4eFMXb4Z08eGjDMD3b9WW3D0XnCLbJpaZ6klz0W0s2tcYSneTBaSs
-RAqxgJgBZ5ZMXtrrHld/5qFoBbStLZLefmcPhnfvamwHDCTLUex8NIAI1u3e9Rhb
-5fbH+gpuYpwHX7hz0FOfpn1sxR03UyxU+ey4AdKe9LG3TJVnB0WcgxpobpbqweLH
-yzcEQCNoFV3r1rlE13Y0aE31/9apoEwiYvqAzEmE38TukDLl/Qg8rkR1t0/lok2P
-G6pWqdN7pmoUovBTvDi5YOthcjZcdOTXXn2Yw4RZVF9uhRsVfku1Eg25SnOje3uY
-smtQLME4eESbePdjyV/okCIle66uHZse+7gNyNmWpf01hM+VmAySIAyKa0Ku8AXZ
-MydEcJTebrNfW9uMLsBx3Ts7z/CBfRng6F8louJGlZtlSwddTkZVcb26T20xeo0a
-ZvdFXM2djTi/a5nbBoZQL85AEeV7HaphFLdPrgmMtS8sSZUEVvdaxp7WJsVuF9cO
-Nxsvx40OYTvfco0W41Lm8/sEuQ7YueEVpZxiv5kX56GTU9vXaOOi+8Z7Ee2w6Adz
-4hrGZkzztggs4tM9geNYnd0XCdZ/ICAskKJABg7biDD1PhEBrqCIqSE3U497vibQ
-Mpkkl/Zpp0BirhGWNyTg8K4JrsQ=
-=d320
------END PGP PUBLIC KEY BLOCK-----
-EOT
-
-rpm --import /root/RPM-GPG-KEY-azulsystems
-rpm --rebuilddb
-
-# Do not include "gpgkey" in the repo definition -- we've already imported it, above.
-cat <<'EOT' > /etc/yum.repos.d/zulu.repo
-[zulu]
-name=zulu-$releasever - Azul Systems Inc., Zulu packages for $basearch
-baseurl=http://repos.azulsystems.com/rhel/$releasever/$basearch
-enabled=1
-gpgcheck=1
-protect=1
-EOT
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@druid.apache.org
For additional commands, e-mail: commits-help@druid.apache.org