You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kylin.apache.org by ni...@apache.org on 2019/12/17 10:24:31 UTC
[kylin] branch master updated: KYLIN-4251 Add livy to docker
This is an automated email from the ASF dual-hosted git repository.
nic pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 99cb664 KYLIN-4251 Add livy to docker
99cb664 is described below
commit 99cb6647f960088cc4099c9c596345f872f4831a
Author: weibin0516 <co...@126.com>
AuthorDate: Wed Nov 13 14:15:53 2019 +0800
KYLIN-4251 Add livy to docker
---
docker/Dockerfile | 24 ++++++++++++++++++++++--
docker/entrypoint.sh | 12 ++++++++++++
docker/run_container.sh | 1 +
3 files changed, 35 insertions(+), 2 deletions(-)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 5be1ae4..06d051e 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -26,16 +26,21 @@ ENV HBASE_VERSION 1.1.2
ENV SPARK_VERSION 2.3.1
ENV ZK_VERSION 3.4.6
ENV KAFKA_VERSION 1.1.1
+ENV LIVY_VERSION 0.6.0
+ENV KYLIN_VERSION 3.0.0-alpha2
ENV JAVA_HOME /home/admin/jdk1.8.0_141
ENV MVN_HOME /home/admin/apache-maven-3.6.1
ENV HADOOP_HOME /home/admin/hadoop-$HADOOP_VERSION
ENV HIVE_HOME /home/admin/apache-hive-$HIVE_VERSION-bin
ENV HADOOP_CONF $HADOOP_HOME/etc/hadoop
+ENV HADOOP_CONF_DIR $HADOOP_HOME/etc/hadoop
ENV HBASE_HOME /home/admin/hbase-$HBASE_VERSION
ENV SPARK_HOME /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6
+ENV SPARK_CONF_DIR /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6/conf
ENV ZK_HOME /home/admin/zookeeper-$ZK_VERSION
ENV KAFKA_HOME /home/admin/kafka_2.11-$KAFKA_VERSION
+ENV LIVY_HOME /home/admin/apache-livy-$LIVY_VERSION-incubating-bin
ENV PATH $PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HIVE_HOME/bin:$HBASE_HOME/bin:$MVN_HOME/bin:spark-$SPARK_VERSION-bin-hadoop2.6/bin:$KAFKA_HOME/bin
USER root
@@ -43,7 +48,7 @@ USER root
WORKDIR /home/admin
# install tools
-RUN yum -y install lsof.x86_64 wget.x86_64 tar.x86_64 git.x86_64 mysql-server.x86_64 mysql.x86_64
+RUN yum -y install lsof.x86_64 wget.x86_64 tar.x86_64 git.x86_64 mysql-server.x86_64 mysql.x86_64 unzip.x86_64
# install mvn
RUN wget http://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.6.1/binaries/apache-maven-3.6.1-bin.tar.gz \
@@ -88,17 +93,32 @@ RUN wget https://archive.apache.org/dist/spark/spark-$SPARK_VERSION/spark-$SPARK
&& rm -f /home/admin/spark-$SPARK_VERSION-bin-hadoop2.6.tgz \
&& cp $HIVE_HOME/conf/hive-site.xml $SPARK_HOME/conf \
&& cp $SPARK_HOME/yarn/*.jar $HADOOP_HOME/share/hadoop/yarn/lib
+RUN cp $HIVE_HOME/lib/mysql-connector-java-5.1.24.jar $SPARK_HOME/jars
+RUN echo spark.sql.catalogImplementation=hive > $SPARK_HOME/conf/spark-defaults.conf
# setup kafka
RUN wget https://archive.apache.org/dist/kafka/$KAFKA_VERSION/kafka_2.11-$KAFKA_VERSION.tgz \
&& tar -zxvf /home/admin/kafka_2.11-$KAFKA_VERSION.tgz \
&& rm -f /home/admin/kafka_2.11-$KAFKA_VERSION.tgz
+# setup livy
+RUN wgethttps://www.apache.org/dist/incubator/livy/$LIVY_VERSION-incubating/apache-livy-$LIVY_VERSION-incubating-bin.zip \
+ && unzip /home/admin/apache-livy-$LIVY_VERSION-incubating-bin.zip \
+ && rm -f /home/admin/apache-livy-$LIVY_VERSION-incubating-bin.zip
+
# copy kylin source code
COPY ./kylin /home/admin/kylin_sourcecode
+
RUN sed -i -e 's#<!--<version>3.4.1</version>-->#<version>3.4.1</version>#g' /home/admin/kylin_sourcecode/pom.xml \
&& echo "kylin.engine.spark-conf.spark.executor.memory=1G" >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
- && echo "kylin.engine.spark-conf-mergedict.spark.executor.memory=1.5G" >> /home/admin/kylin_sourcecode/build/conf/kylin.properties
+ && echo "kylin.engine.spark-conf-mergedict.spark.executor.memory=1.5G" >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
+ && echo "kylin.engine.livy-conf.livy-url=http://127.0.0.1:8998" >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
+ && echo kylin.engine.livy-conf.livy-key.file=hdfs://localhost:9000/kylin/livy/kylin-job-$KYLIN_VERSION.jar >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
+ && echo kylin.engine.livy-conf.livy-arr.jars=hdfs://localhost:9000/kylin/livy/hbase-client-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-common-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-hadoop-compat-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-hadoop2-compat-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/hbase-server-$HBASE_VERSION.jar,hdfs://localhost:9000/kylin/livy/htrace-core-*-incubating.jar,hdfs://localhost:9000/kylin/livy/metrics- [...]
+ && echo kylin.source.hive.quote-enabled=false >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
+ && echo kylin.engine.spark-conf.spark.eventLog.dir=hdfs://localhost:9000/kylin/spark-history >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
+ && echo kylin.engine.spark-conf.spark.history.fs.logDirectory=hdfs://localhost:9000/kylin/spark-history >> /home/admin/kylin_sourcecode/build/conf/kylin.properties \
+ && echo kylin.source.hive.redistribute-flat-table=false >> /home/admin/kylin_sourcecode/build/conf/kylin.properties
COPY ./entrypoint.sh /home/admin/entrypoint.sh
RUN chmod u+x /home/admin/entrypoint.sh
diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 28fb3b5..9c03715 100644
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -47,6 +47,18 @@ rm -rf /tmp/kafka-logs
rm -rf /data/zookeeper/*
nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties &
+# start livy
+hdfs dfs -mkdir -p /kylin/livy
+hdfs dfs -put -f $HBASE_HOME/lib/hbase-client-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+hdfs dfs -put -f $HBASE_HOME/lib/hbase-common-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+hdfs dfs -put -f $HBASE_HOME/lib/hbase-hadoop-compat-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+hdfs dfs -put -f $HBASE_HOME/lib/hbase-hadoop2-compat-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+hdfs dfs -put -f $HBASE_HOME/lib/hbase-server-$HBASE_VERSION.jar hdfs://localhost:9000/kylin/livy/
+hdfs dfs -put -f $HBASE_HOME/lib/htrace-core-*-incubating.jar hdfs://localhost:9000/kylin/livy/
+hdfs dfs -put -f $HBASE_HOME/lib/metrics-core-*.jar hdfs://localhost:9000/kylin/livy/
+hdfs dfs -put -f $KYLIN_HOME/lib/kylin-job-$KYLIN_VERSION.jar hdfs://localhost:9000/kylin/livy/
+$LIVY_HOME/bin/livy-server start
+
# prepare kafka topic and data
$KAFKA_HOME/bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic kylin_streaming_topic
diff --git a/docker/run_container.sh b/docker/run_container.sh
index a0acc2f..5213c55 100644
--- a/docker/run_container.sh
+++ b/docker/run_container.sh
@@ -23,4 +23,5 @@ docker run -d \
-p 8032:8032 \
-p 8042:8042 \
-p 60010:60010 \
+-p 8998:8998 \
apache-kylin-standalone
\ No newline at end of file