You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2019/09/25 20:19:39 UTC
[incubator-gobblin] branch master updated: [GOBBLIN-883] Add docker
files and compose
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new df2be85 [GOBBLIN-883] Add docker files and compose
df2be85 is described below
commit df2be85e446fb26dc27682f88fffebead80e946b
Author: William Lo <wl...@wlo-mn1.linkedin.biz>
AuthorDate: Wed Sep 25 13:19:32 2019 -0700
[GOBBLIN-883] Add docker files and compose
Closes #2736 from Will-Lo/docker-gaas
---
.dockerignore | 25 +++++
.../gobblin-service/alpine-gaas-latest/Dockerfile | 18 +++
.../alpine-gaas-latest/docker-compose.yml | 8 ++
.../alpine-gaas-latest/entrypoint.sh | 122 +++++++++++++++++++++
.../user-guide/Building-Gobblin-as-a-Service.md | 28 +++++
5 files changed, 201 insertions(+)
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..7157a84
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,25 @@
+.classpath
+.codecov_bash
+.dockerignore
+.github
+.gradle
+.git
+.gitignore
+.project
+*.md
+
+gobblin-docker/
+gobblin-docs/
+
+# Intellij related
+**/.idea
+**/*.iml
+**/*.iws
+**/*.ipr
+.shelf/
+
+travis/
+temp/
+
+# allow entrypoint scripts
+!/gobblin-docker/**/*.sh
diff --git a/gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile b/gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
new file mode 100644
index 0000000..be81ebb
--- /dev/null
+++ b/gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
@@ -0,0 +1,18 @@
+ARG GOBBLIN_ROOT=${GOBBLIN_WORK_DIR}
+
+FROM openjdk:8-alpine as build
+WORKDIR /home/gobblin
+COPY ${GOBBLIN_ROOT} .
+RUN apk add --no-cache bash && \
+ ./gradlew :gobblin-distribution:buildDistributionTar -Pversion=latest
+
+FROM openjdk:8-jre-alpine
+ARG GOBBLIN_VERSION=latest
+WORKDIR /home/gobblin
+COPY ${GOBBLIN_ROOT}/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh ./gobblin-dist/bin/entrypoint.sh
+COPY --from=build /home/gobblin/apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz .
+RUN tar -xvf apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz && \
+ mkdir /tmp/templateCatalog/ && \
+ chmod +x ./gobblin-dist/bin/gobblin-service.sh && \
+ rm apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz
+ENTRYPOINT ["./gobblin-dist/bin/entrypoint.sh", "start"]
\ No newline at end of file
diff --git a/gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml b/gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml
new file mode 100644
index 0000000..82607f7
--- /dev/null
+++ b/gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml
@@ -0,0 +1,8 @@
+version: '3'
+services:
+ gobblin-service:
+ build:
+ context: ../../../
+ dockerfile: gobblin-docker/gobblin-service/alpine-gaas-latest/Dockerfile
+ ports:
+ - 6956:6956
\ No newline at end of file
diff --git a/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh b/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
new file mode 100755
index 0000000..75a5426
--- /dev/null
+++ b/gobblin-docker/gobblin-service/alpine-gaas-latest/entrypoint.sh
@@ -0,0 +1,122 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Print an error message and exit
+
+function start() {
+ for jarFile in `ls ${FWDIR_LIB}/*`
+ do
+ GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
+ done
+
+ export HADOOP_USER_CLASSPATH_FIRST=true
+
+ CLASSPATH=${FWDIR_CONF}:${GOBBLIN_JARS}:${SERVICE_CONF_DIR}:${HADOOP_HOME}/lib
+ if [ -n "$EXTRA_JARS" ]; then
+ CLASSPATH=$CLASSPATH:"$EXTRA_JARS"
+ fi
+
+ LOG_ARGS="1>${FWDIR_LOGS}/GobblinService.stdout 2>${FWDIR_LOGS}/GobblinService.stderr"
+
+# LOG4J_ARGS="-Dlog4j.configuration=conf/log4j.xml"
+ DEBUGGER_ARGS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006"
+
+ LOG4J_PATH=file://${FWDIR_CONF}/log4j-service.properties
+ $JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH $JVM_FLAGS $LOG4J_ARGS $DEBUGGER_ARGS org.apache.gobblin.service.modules.core.GobblinServiceManager --service_name $SERVICE_NAME $LOG_ARGS
+}
+
+function stop() {
+ if [ -f "$PID" ]; then
+ if kill -0 $PID_VALUE > /dev/null 2>&1; then
+ echo 'Stopping the Gobblin service'
+ kill $PID_VALUE
+ else
+ echo "Process $PID_VALUE is not running"
+ fi
+ else
+ echo "No pid file found"
+ fi
+}
+
+FWDIR="$(cd `dirname $0`/..; pwd)"
+FWDIR_LIB=${FWDIR}/lib
+FWDIR_CONF=${FWDIR}/conf/service
+FWDIR_BIN=${FWDIR}/bin
+FWDIR_LOGS=${FWDIR}/logs
+SERVICE_NAME="gobblin_service"
+
+. ${FWDIR_BIN}/gobblin-env.sh
+
+for i in "$@"
+do
+ case "$1" in
+ start|stop)
+ ACTION="$1"
+ ;;
+ --jvmflags)
+ JVM_FLAGS="$2"
+ shift
+ ;;
+ --jars)
+ EXTRA_JARS="$2"
+ shift
+ ;;
+ --service)
+ SERVICE_NAME="$2"
+ shift
+ ;;
+ --help)
+ print_usage
+ exit 0
+ ;;
+ *)
+ ;;
+ esac
+ shift
+done
+
+if [ -z "$JAVA_HOME" ]; then
+ die "Environment variable JAVA_HOME not set!"
+fi
+
+# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
+if [ -n "$JVM_FLAGS" ]; then
+ JVM_FLAGS="-Xmx1g -Xms512m"
+fi
+
+PID="$FWDIR/.gobblin-service-app-pid"
+
+if [ -f "$PID" ]; then
+ PID_VALUE=`cat $PID` > /dev/null 2>&1
+else
+ PID_VALUE=""
+fi
+
+case "$ACTION" in
+ "start")
+ start
+ ;;
+ "stop")
+ stop
+ ;;
+ *)
+ print_usage
+ exit 1
+ ;;
+esac
+
diff --git a/gobblin-docs/user-guide/Building-Gobblin-as-a-Service.md b/gobblin-docs/user-guide/Building-Gobblin-as-a-Service.md
new file mode 100644
index 0000000..857680d
--- /dev/null
+++ b/gobblin-docs/user-guide/Building-Gobblin-as-a-Service.md
@@ -0,0 +1,28 @@
+Table of Contents
+--------------------
+
+[TOC]
+
+
+# Introduction
+Gobblin as a service is a service that takes in a user request (a logical flow) and converts it into a series of Gobblin Jobs, and monitors these jobs in a distributed manner.
+The design of the service can be found here: https://cwiki.apache.org/confluence/display/GOBBLIN/Gobblin+as+a+Service
+
+# Running Gobblin as a Service
+1. [Build Gobblin] (./Building-Gobblin.md) or use one of the [provided distributions] (https://github.com/apache/incubator-gobblin/releases)
+2. Untar the build file `tar -xvf apache-gobblin-incubating-bin-${GOBBLIN_VERSION}.tar.gz`
+3. Execute the start script `./gobblin-dist/bin/gobblin-service.sh`
+4. View output in `service.out`
+
+Currently the setup only runs a portion of the service, but work will be done to have a basic end-to-end workflow soon.
+
+The service can now be accessed on `localhost:6956`
+
+# Running Gobblin as a Service with Docker
+There is also a Dockerfile to create new images of Gobblin based on the source code that can be easily run.
+
+1. `export GOBBLIN_WORK_DIR=<root_directory_of_gobblin>`
+2. `docker compose -f gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml build`
+3. `docker compose -f gobblin-docker/gobblin-service/alpine-gaas-latest/docker-compose.yml up`
+
+The docker container exposes the endpoints which can be accessed on `localhost:6956`
\ No newline at end of file