You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by fo...@apache.org on 2018/08/19 18:25:17 UTC
[incubator-airflow-ci] 16/34: Install Hadoop, Hive and MiniCluster
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-airflow-ci.git
commit 62b4ef62c53d5dbc3b0a09270f8c52b8245b78e7
Author: Gerardo Curiel <ge...@rea-group.com>
AuthorDate: Sat May 19 09:58:12 2018 +1000
Install Hadoop, Hive and MiniCluster
---
Dockerfile.base | 28 ++++++++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git a/Dockerfile.base b/Dockerfile.base
index dd0a220..95a2281 100644
--- a/Dockerfile.base
+++ b/Dockerfile.base
@@ -27,6 +27,16 @@ ENV LC_ALL en_US.UTF-8
ENV LC_CTYPE en_US.UTF-8
ENV LC_MESSAGES en_US.UTF-8
+ENV HADOOP_VERSION 2.6.0
+ENV HADOOP_DISTRO=cdh
+ENV HADOOP_HOME=/tmp/hadoop-${HADOOP_DISTRO}
+ENV HIVE_HOME=/tmp/hive
+
+RUN mkdir ${HADOOP_HOME} && \
+ mkdir ${HIVE_HOME} && \
+ mkdir /tmp/minicluster && \
+ mkdir /user/hive/warehouse
+
RUN apt-get update && apt-get install -y \
openjdk-8-jdk \
wget curl \
@@ -50,6 +60,24 @@ RUN apt-get update && apt-get install -y \
&& locale-gen \
&& update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8
+# Install Hadoop
+RUN cd /tmp && \
+ wget -q https://archive.cloudera.com/cdh5/cdh/5/hadoop-${HADOOP_VERSION}-cdh5.11.0.tar.gz && \
+ tar xzf hadoop-${HADOOP_VERSION}-cdh5.11.0.tar.gz --strip-components 1 -C $HADOOP_HOME && \
+ rm hadoop-${HADOOP_VERSION}-cdh5.11.0.tar.gz
+
+# Install Hive
+RUN cd /tmp && \
+ wget -q https://archive.cloudera.com/cdh5/cdh/5/hive-1.1.0-cdh5.11.0.tar.gz && \
+ tar xzf hive-1.1.0-cdh5.11.0.tar.gz --strip-components 1 -C $HIVE_HOME && \
+ rm hive-1.1.0-cdh5.11.0.tar.gz
+
+# Install MiniCluster
+RUN cd /tmp && \
+ wget -q https://github.com/bolkedebruin/minicluster/releases/download/1.1/minicluster-1.1-SNAPSHOT-bin.zip && \
+ unzip minicluster-1.1-SNAPSHOT-bin.zip -d /tmp
+ rm minicluster-1.1-SNAPSHOT-bin.zip
+
RUN pip install --upgrade pip && \
hash -r pip && \
pip install tox codecov