You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gobblin.apache.org by su...@apache.org on 2019/09/27 20:37:50 UTC
[incubator-gobblin] branch master updated:
[GOBBLIN-707][GOBBLIN-694][GOBBLIN-477][GOBBLIN-719] rewrite gobblin script
to combine all modes and command
This is an automated email from the ASF dual-hosted git repository.
suvasude pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new b9bcfa0 [GOBBLIN-707][GOBBLIN-694][GOBBLIN-477][GOBBLIN-719] rewrite gobblin script to combine all modes and command
b9bcfa0 is described below
commit b9bcfa0100b13dc02dbb2c3d2e62fb536be6959c
Author: Jay Sen <js...@paypal.com>
AuthorDate: Fri Sep 27 13:37:41 2019 -0700
[GOBBLIN-707][GOBBLIN-694][GOBBLIN-477][GOBBLIN-719] rewrite gobblin script to combine all modes and command
[GOBBLIN-719] fix invalid git links for classes in
docs
fixing all file paths and linksA
[GOBBLIN-707]-rewrite gobblin script combining all
gobblin modes and commands along with configs
updating as per review 1
updates docs as per gobblin.sh changes + some more
improvements in gobblin.sh
updates docs as per gobblin.sh changes + some more
improvements in gobblin.sh - 2
Merge branch 'master' of
github.com:jhsenjaliya/incubator-gobblin
Merge branch 'master' of
github.com:apache/incubator-gobblin
Merge branch 'master' of
github.com:apache/incubator-gobblin
[GOBBLIN-707]-rewrite gobblin script combining all
gobblin modes and commands along with configs
updating as per review 1
updates docs as per gobblin.sh changes + some more
improvements in gobblin.sh
updates docs as per gobblin.sh changes + some more
improvements in gobblin.sh - 2
Merge branch 'GOBBLIN-707' of
github.com:jhsenjaliya/incubator-gobblin into
GOBBLIN-707
adding project version
Merge branch 'master' of
github.com:apache/incubator-gobblin into
GOBBLIN-707
arrange HADOOP classpath
restructure command vs services help
all command and cli classes under GobblinCli with
Alias, remove Cli class
uncomment required code
keep individual script that calls new gobblin
script internally
resolves findBug error, replaced system exit with
return
update in config, keeping default as before
update to yarn, MR config
updating gobblin.sh and adding gobblin-service
config
Merge branch 'master' into GOBBLIN-707
updates to gobblin.sh and help on cli classes
rename service manager to gobblin-as-service
merge upstream changes
revert to upstream config to put change later
Closes #2578 from jhsenjaliya/GOBBLIN-707
---
CHANGELOG.md | 4 +-
bin/gobblin-admin.sh | 125 +----
bin/gobblin-aws.sh | 114 +---
bin/gobblin-cluster-master.sh | 119 +---
bin/gobblin-cluster-worker.sh | 126 +----
bin/gobblin-mapreduce.sh | 196 +------
bin/gobblin-service.sh | 122 +---
bin/gobblin-standalone.sh | 247 +--------
bin/gobblin-yarn.sh | 112 +---
bin/gobblin.sh | 611 +++++++++++++++++++--
.../gobblin_password_encryptor.sh | 11 +-
bin/historystore-manager.sh | 43 +-
bin/statestore-checker.sh | 17 +-
bin/statestore-cleaner.sh | 2 +
conf/aws/application.conf | 8 +-
.../log4j-yarn.properties => aws/log4j.properties} | 0
conf/{ => cli}/gobblin-cli.properties | 0
.../log4j.properties} | 2 +
.../worker => cluster-master}/application.conf | 27 +
.../log4j.properties} | 1 +
.../worker => cluster-master}/quartz.properties | 0
.../master => cluster-worker}/application.conf | 23 +
.../log4j.properties} | 0
.../master => cluster-worker}/quartz.properties | 0
conf/{ => compaction}/log4j-compaction.xml | 0
.../application.conf | 5 +-
.../log4j.properties} | 0
.../quartz.properties | 0
conf/gobblin-standalone-v2.properties | 65 ---
conf/gobblin-standalone.properties | 68 ---
conf/log4j-mapreduce.xml | 60 --
conf/log4j-standalone.xml | 50 --
conf/log4j.properties | 25 -
conf/log4j.xml | 35 --
.../application.conf} | 18 +-
.../log4j.properties} | 5 +-
conf/service/reference.conf | 5 -
conf/standalone/application.conf | 27 +-
.../log4j-cluster.properties => log4j.properties} | 6 +-
conf/standalone/master/reference.conf | 15 -
conf/standalone/reference.conf | 15 -
conf/standalone/worker/reference.conf | 15 -
conf/yarn/application.conf | 39 +-
.../log4j-aws.properties => yarn/log4j.properties} | 0
conf/yarn/quartz.properties | 21 +-
conf/yarn/reference.conf | 43 --
.../src/main/java/org/apache/gobblin/cli/Cli.java | 196 -------
.../main/java/org/apache/gobblin/cli/Command.java | 24 -
.../java/org/apache/gobblin/cli/JobCommand.java | 49 +-
gobblin-docs/Getting-Started.md | 23 +-
gobblin-docs/Gobblin-Architecture.md | 3 +-
gobblin-docs/case-studies/Kafka-HDFS-Ingestion.md | 4 +-
gobblin-docs/case-studies/Publishing-Data-to-S3.md | 4 +-
gobblin-docs/sinks/AvroHdfsDataWriter.md | 2 +-
gobblin-docs/user-guide/Gobblin-CLI.md | 206 +++++--
gobblin-docs/user-guide/Gobblin-Deployment.md | 122 +---
.../util/DatabaseJobHistoryStoreSchemaManager.java | 10 +-
.../gobblin/runtime/cli/CliEmbeddedGobblin.java | 2 +-
.../runtime/util/JobStateToJsonConverter.java | 14 +-
.../org/apache/gobblin/runtime/cli/GobblinCli.java | 7 +-
60 files changed, 1046 insertions(+), 2047 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7bce25d..f141955 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1172,9 +1172,9 @@ GOBBLIN 0.6.0
NEW FEATURES
* [Compaction] Added M/R compaction/de-duping for hourly data
-* [Compaction] Added late data handling for hourly and daily M/R compaction: https://github.com/linkedin/gobblin/wiki/Compaction#handling-late-records; added support for triggering M/R compaction if late data exceeds a threshold
+* [Compaction] Added late data handling for hourly and daily M/R compaction: https://github.com/apache/incubator-gobblin/wiki/Compaction#handling-late-records; added support for triggering M/R compaction if late data exceeds a threshold
* [I/O] Added support for using Hive SerDe's through HiveWritableHdfsDataWriter
-* [I/O] Added the concept of data partitioning to writers: https://github.com/linkedin/gobblin/wiki/Partitioned-Writers
+* [I/O] Added the concept of data partitioning to writers: https://github.com/apache/incubator-gobblin/wiki/Partitioned-Writers
* [Runtime] Added CliLocalJobLauncher for launching single jobs from the command line.
* [Converters] Added AvroSchemaFieldRemover that can remove specific fields from a (possibly recursive) Avro schema.
* [DQ] Added new row-level policies RecordTimestampLowerBoundPolicy and AvroRecordTimestampLowerBoundPolicy for checking if a record timestamp is too far in the past.
diff --git a/bin/gobblin-admin.sh b/bin/gobblin-admin.sh
index 0a48e2d..1a510db 100755
--- a/bin/gobblin-admin.sh
+++ b/bin/gobblin-admin.sh
@@ -17,126 +17,7 @@
# limitations under the License.
#
-function print_usage() {
- echo "gobblin-admin.sh [JAVA_OPTION] COMMAND [OPTION]"
- echo "Where JAVA_OPTION can be:"
- echo " --fwdir <fwd dir> Gobblin's dist directory: if not set, taken from \${GOBBLIN_FWDIR}"
- echo " --logdir <log dir> Gobblin's log directory: if not set, taken from \${GOBBLIN_LOG_DIR}"
- echo " --jars <comma-separated list of job jars> Job jar(s): if not set, \${GOBBLIN_FWDIR/lib} is examined"
- echo " --help Display this help and exit"
- echo "COMMAND is one of the following:"
- echo " jobs|tasks"
- echo "And OPTION are any options associated with the command, as specified by the CLI."
-}
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
-
-for i in "$@"
-do
- case "$1" in
- jobs|tasks)
- ACTION="$1"
- break
- ;;
- --fwdir)
- FWDIR="$2"
- shift 2
- ;;
- --logdir)
- LOG_DIR="$2"
- shift 2
- ;;
- --jars)
- JARS="$2"
- shift 2
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
-done
-
-if [ -z "$ACTION" ]; then
- print_usage
- exit 0
-fi
-
-# Source gobblin default vars
-[ -f /etc/default/gobblin ] && . /etc/default/gobblin
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-if [ -n "$FWDIR" ]; then
- export GOBBLIN_FWDIR="$FWDIR"
-fi
-
-if [ -z "$GOBBLIN_FWDIR" ]; then
- die "Environment variable FWDIR not set!"
-fi
-
-FWDIR_LIB=$GOBBLIN_FWDIR/lib
-FWDIR_CONF=$GOBBLIN_FWDIR/conf
-
-# User defined log directory overrides $GOBBLIN_LOG_DIR
-if [ -n "$LOG_DIR" ]; then
- export GOBBLIN_LOG_DIR="$LOG_DIR"
-fi
-
-if [ -z "$GOBBLIN_LOG_DIR" ]; then
- die "GOBBLIN_LOG_DIR is not set!"
-fi
-
-CONFIG_FILE=$FWDIR_CONF/gobblin-standalone.properties
-
-set_user_jars(){
- local separator=''
- if [ -n "$1" ]; then
- IFS=','
- read -ra userjars <<< "$1"
- for userjar in ${userjars[@]}; do
- add_user_jar "$userjar"
- done
- unset IFS
- fi
-}
-
-add_user_jar(){
- local dirname=`dirname "$1"`
- local jarname=`basename "$1"`
- dirname=`cd "$dirname">/dev/null; pwd`
- GOBBLIN_JARS+="$separator$dirname/$jarname"
- separator=':'
-}
-
-# Add the absoulte path of the user defined job jars to the GOBBLIN_JARS
-set_user_jars "$JARS"
-
-for jar in $(ls -d $FWDIR_LIB/*); do
- if [ "$GOBBLIN_JARS" != "" ]; then
- GOBBLIN_JARS+=":$jar"
- else
- GOBBLIN_JARS=$jar
- fi
-done
-
-CLASSPATH=":$GOBBLIN_JARS:$FWDIR_CONF"
-
-COMMAND="$JAVA_HOME/bin/java -Xmx1024m -Xms256m "
-COMMAND+="-XX:+UseCompressedOops "
-COMMAND+="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$GOBBLIN_LOG_DIR/ "
-COMMAND+="-Xloggc:$GOBBLIN_LOG_DIR/gobblin-gc.log "
-COMMAND+="-Dgobblin.logs.dir=$GOBBLIN_LOG_DIR "
-COMMAND+="-Dlog4j.configuration=file://$FWDIR_CONF/log4j-standalone.xml "
-COMMAND+="-cp $CLASSPATH "
-COMMAND+="org.apache.gobblin.cli.Cli $@"
-$COMMAND
\ No newline at end of file
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin cli $@
\ No newline at end of file
diff --git a/bin/gobblin-aws.sh b/bin/gobblin-aws.sh
index c3ac2f8..48fd15f 100755
--- a/bin/gobblin-aws.sh
+++ b/bin/gobblin-aws.sh
@@ -17,115 +17,7 @@
# limitations under the License.
#
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
-
-function print_usage() {
- echo "gobblin-aws.sh <start | stop>"
- echo "Where OPTION can be:"
- echo " --jvmflags <string of jvm flags> String containing any additional JVM flags to include"
- echo " --jars <column-separated list of extra jars> Column-separated list of extra jars to put on the CLASSPATH"
- echo " --help Display this help and exit"
-}
-
-function start() {
- for jarFile in `ls ${FWDIR_LIB}/*`
- do
- GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
- done
-
- export HADOOP_USER_CLASSPATH_FIRST=true
-
- CLASSPATH=${FWDIR_CONF}:${GOBBLIN_JARS}:${AWS_CONF_DIR}:${HADOOP_AWS_HOME}/lib
- if [ -n "$EXTRA_JARS" ]; then
- CLASSPATH=$CLASSPATH:"$EXTRA_JARS"
- fi
-
- LOG4J_PATH=file://${FWDIR_CONF}/log4j-aws.properties
- COMMAND="$JAVA_HOME/bin/java -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.aws.GobblinAWSClusterLauncher -D log4j.configuration=$LOG4J_PATH"
-
- echo "Running command:"
- echo "$COMMAND"
- nohup $COMMAND & echo $! > $PID
-}
-
-function stop() {
- if [ -f "$PID" ]; then
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo 'Stopping the Gobblin AWS application'
- kill $PID_VALUE
- else
- echo "Process $PID_VALUE is not running"
- fi
- else
- echo "No pid file found"
- fi
-}
-
-FWDIR="$(cd `dirname $0`/..; pwd)"
-FWDIR_LIB=${FWDIR}/lib
-FWDIR_CONF=${FWDIR}/conf/aws
-FWDIR_BIN=${FWDIR}/bin
-
-. ${FWDIR_BIN}/gobblin-env.sh
-
-for i in "$@"
-do
- case "$1" in
- start|stop)
- ACTION="$1"
- shift
- ;;
- --jvmflags)
- JVM_FLAGS="$1"
- shift
- ;;
- --jars)
- EXTRA_JARS="$1"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
-if [ -n "$JVM_FLAGS" ]; then
- JVM_FLAGS="-Xmx1g -Xms512m"
-fi
-
-PID="$FWDIR/.gobblin-aws-app-pid"
-
-if [ -f "$PID" ]; then
- PID_VALUE=`cat $PID` > /dev/null 2>&1
-else
- PID_VALUE=""
-fi
-
-case "$ACTION" in
- "start")
- start
- ;;
- "stop")
- stop
- ;;
- *)
- print_usage
- exit 1
- ;;
-esac
-
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin service aws $@
\ No newline at end of file
diff --git a/bin/gobblin-cluster-master.sh b/bin/gobblin-cluster-master.sh
index 131c1cc..cc39264 100755
--- a/bin/gobblin-cluster-master.sh
+++ b/bin/gobblin-cluster-master.sh
@@ -17,120 +17,7 @@
# limitations under the License.
#
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
-function print_usage() {
- echo "gobblin-cluster-master.sh <start | stop>"
- echo "Where OPTION can be:"
- echo " --jvmflags <string of jvm flags> String containing any additional JVM flags to include"
- echo " --jars <column-separated list of extra jars> Column-separated list of extra jars to put on the CLASSPATH"
- echo " --help Display this help and exit"
-}
-
-function start() {
- for jarFile in `ls ${FWDIR_LIB}/*`
- do
- GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
- done
-
- export HADOOP_USER_CLASSPATH_FIRST=true
-
- CLASSPATH=${FWDIR_CONF}:${GOBBLIN_JARS}:${CLUSTER_CONF_DIR}:${HADOOP_HOME}/lib
- if [ -n "$EXTRA_JARS" ]; then
- CLASSPATH=$CLASSPATH:"$EXTRA_JARS"
- fi
-
- LOG_ARGS="1>${FWDIR_LOGS}/GobblinCluster.master.stdout 2>${FWDIR_LOGS}/GobblinCluster.master.stderr"
-
- LOG4J_PATH=file://${FWDIR_CONF}/log4j-cluster.properties
- COMMAND="$JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.cluster.GobblinClusterManager --standalone_cluster true --app_name $CLUSTER_NAME $LOG_ARGS"
-
- echo "Running command:"
- echo "$COMMAND"
- nohup $COMMAND >clustermaster.out 2>&1 & echo $! > $PID
-}
-
-function stop() {
- if [ -f "$PID" ]; then
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo 'Stopping the Gobblin Standalone cluster'
- kill $PID_VALUE
- else
- echo "Process $PID_VALUE is not running"
- fi
- else
- echo "No pid file found"
- fi
-}
-
-FWDIR="$(cd `dirname $0`/..; pwd)"
-FWDIR_LIB=${FWDIR}/lib
-FWDIR_CONF=${FWDIR}/conf/standalone/master
-FWDIR_BIN=${FWDIR}/bin
-FWDIR_LOGS=${FWDIR}/logs
-CLUSTER_NAME="standalone_cluster"
-
-. ${FWDIR_BIN}/gobblin-env.sh
-
-for i in "$@"
-do
- case "$1" in
- start|stop)
- ACTION="$1"
- ;;
- --jvmflags)
- JVM_FLAGS="$2"
- shift
- ;;
- --jars)
- EXTRA_JARS="$2"
- shift
- ;;
- --cluster)
- CLUSTER_NAME="$2"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
-if [ -n "$JVM_FLAGS" ]; then
- JVM_FLAGS="-Xmx1g -Xms512m"
-fi
-
-PID="$FWDIR/.gobblin-cluster-master-app-pid"
-
-if [ -f "$PID" ]; then
- PID_VALUE=`cat $PID` > /dev/null 2>&1
-else
- PID_VALUE=""
-fi
-
-case "$ACTION" in
- "start")
- start
- ;;
- "stop")
- stop
- ;;
- *)
- print_usage
- exit 1
- ;;
-esac
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin service cluster-master $@
\ No newline at end of file
diff --git a/bin/gobblin-cluster-worker.sh b/bin/gobblin-cluster-worker.sh
index 80ed67a..ec99c83 100755
--- a/bin/gobblin-cluster-worker.sh
+++ b/bin/gobblin-cluster-worker.sh
@@ -17,127 +17,7 @@
# limitations under the License.
#
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
-function print_usage() {
- echo "gobblin-cluster-worker.sh <start | stop>"
- echo "Where OPTION can be:"
- echo " --cluster <unique name for the cluster> Unique name for the cluster"
- echo " --workerid <unique id of workers> Worker id for each worker if running multiple workers"
- echo " --jvmflags <string of jvm flags> String containing any additional JVM flags to include"
- echo " --jars <column-separated list of extra jars> Column-separated list of extra jars to put on the CLASSPATH"
- echo " --help Display this help and exit"
-}
-
-function start() {
- for jarFile in `ls ${FWDIR_LIB}/*`
- do
- GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
- done
-
- export HADOOP_USER_CLASSPATH_FIRST=true
-
- CLASSPATH=${FWDIR_CONF}:${GOBBLIN_JARS}:${CLUSTER_CONF_DIR}:${HADOOP_HOME}/lib
- if [ -n "$EXTRA_JARS" ]; then
- CLASSPATH=$CLASSPATH:"$EXTRA_JARS"
- fi
-
- LOG_ARGS="1>${FWDIR_LOGS}/GobblinCluster.worker.$WORKER_ID.stdout 2>${FWDIR_LOGS}/GobblinCluster.worker.$WORKER_ID.stderr"
-
- LOG4J_PATH=file://${FWDIR_CONF}/log4j-cluster.properties
- COMMAND="$JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.cluster.GobblinTaskRunner --app_name $CLUSTER_NAME --helix_instance_name worker.$WORKER_ID $LOG_ARGS"
-
- echo "Running command:"
- echo "$COMMAND"
- nohup $COMMAND >worker.$WORKER_ID.out 2>&1 & echo $! > $PID
-}
-
-function stop() {
- if [ -f "$PID" ]; then
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo 'Stopping the Gobblin Standalone cluster'
- kill $PID_VALUE
- else
- echo "Process $PID_VALUE is not running"
- fi
- else
- echo "No pid file found"
- fi
-}
-
-FWDIR="$(cd `dirname $0`/..; pwd)"
-FWDIR_LIB=${FWDIR}/lib
-FWDIR_CONF=${FWDIR}/conf/standalone/worker
-FWDIR_BIN=${FWDIR}/bin
-FWDIR_LOGS=${FWDIR}/logs
-CLUSTER_NAME="standalone_cluster"
-WORKER_ID="single"
-
-. ${FWDIR_BIN}/gobblin-env.sh
-
-for i in "$@"
-do
- case "$1" in
- start|stop)
- ACTION="$1"
- ;;
- --jvmflags)
- JVM_FLAGS="$2"
- shift
- ;;
- --jars)
- EXTRA_JARS="$2"
- shift
- ;;
- --cluster)
- CLUSTER_NAME="$2"
- shift
- ;;
- --workerid)
- WORKER_ID="$2"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
-if [ -n "$JVM_FLAGS" ]; then
- JVM_FLAGS="-Xmx1g -Xms512m"
-fi
-
-PID="$FWDIR/.gobblin-cluster-worker-app-pid.$WORKER_ID"
-
-if [ -f "$PID" ]; then
- PID_VALUE=`cat $PID` > /dev/null 2>&1
-else
- PID_VALUE=""
-fi
-
-case "$ACTION" in
- "start")
- start
- ;;
- "stop")
- stop
- ;;
- *)
- print_usage
- exit 1
- ;;
-esac
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin service cluster-worker $@
diff --git a/bin/gobblin-mapreduce.sh b/bin/gobblin-mapreduce.sh
index da8b51d..5e050fa 100755
--- a/bin/gobblin-mapreduce.sh
+++ b/bin/gobblin-mapreduce.sh
@@ -17,199 +17,11 @@
# limitations under the License.
#
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
+
##############################################################
############### Run Gobblin Jobs on Hadoop MR ################
##############################################################
-# Set during the distribution build
-GOBBLIN_VERSION=@project.version@
-
-FWDIR="$(cd `dirname $0`/..; pwd)"
-FWDIR_LIB=$FWDIR/lib
-FWDIR_CONF=$FWDIR/conf
-FWDIR_BIN=$FWDIR/bin
-
-function print_usage(){
- echo "Usage: gobblin-mapreduce.sh [OPTION] --conf <job configuration file>"
- echo "Where OPTION can be:"
- echo " --jt <job tracker / resource manager URL> Job submission URL: if not set, taken from \${HADOOP_HOME}/conf"
- echo " --fs <file system URL> Target file system: if not set, taken from \${HADOOP_HOME}/conf"
- echo " --jars <comma-separated list of job jars> Job jar(s): if not set, \"$FWDIR_LIB\" is examined"
- echo " --workdir <job work dir> Gobblin's base work directory: if not set, taken from \${GOBBLIN_WORK_DIR}"
- echo " --projectversion <version> Gobblin version to be used. If set, overrides the distribution build version"
- echo " --logdir <log dir> Gobblin's log directory: if not set, taken from \${GOBBLIN_LOG_DIR} if present. Otherwise \"$FWDIR/logs\" is used"
- echo " --help Display this help and exit"
-}
-
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
-
-for i in "$@"
-do
- case "$1" in
- --jt)
- JOB_TRACKER_URL="$2"
- shift
- ;;
- --fs)
- FS_URL="$2"
- shift
- ;;
- --jars)
- JARS="$2"
- shift
- ;;
- --workdir)
- WORK_DIR="$2"
- shift
- ;;
- --logdir)
- LOG_DIR="$2"
- shift
- ;;
- --conf)
- JOB_CONFIG_FILE="$2"
- shift
- ;;
- --projectversion)
- GOBBLIN_VERSION="$2"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if ( [ -z "$GOBBLIN_VERSION" ] || [ "$GOBBLIN_VERSION" == "@project.version@" ] ); then
- die "Gobblin project version is not set!"
-fi
-
-if [ -z "$JOB_CONFIG_FILE" ]; then
- die "No job configuration file set!"
-fi
-
-# User defined work directory overrides $GOBBLIN_WORK_DIR
-if [ -n "$WORK_DIR" ]; then
- export GOBBLIN_WORK_DIR="$WORK_DIR"
-fi
-
-if [ -z "$GOBBLIN_WORK_DIR" ]; then
- die "GOBBLIN_WORK_DIR is not set!"
-fi
-
-# User defined log directory overrides $GOBBLIN_LOG_DIR
-if [ -n "$LOG_DIR" ]; then
- export GOBBLIN_LOG_DIR="$LOG_DIR"
-fi
-
-if [ -z "$GOBBLIN_LOG_DIR" ]; then
- GOBBLIN_LOG_DIR="$FWDIR/logs"
-fi
-
-. $FWDIR_BIN/gobblin-env.sh
-
-USER_JARS=""
-separator=''
-set_user_jars(){
- if [ -n "$1" ]; then
- IFS=','
- read -ra userjars <<< "$1"
- for userjar in ${userjars[@]}; do
- add_user_jar "$userjar"
- done
- unset IFS
- fi
-}
-
-add_user_jar(){
- local dirname=`dirname "$1"`
- local jarname=`basename "$1"`
- dirname=`cd "$dirname">/dev/null; pwd`
- USER_JARS+="$separator$dirname/$jarname"
- separator=','
-}
-
-# Add the absolute path of the user defined job jars to the LIBJARS first
-set_user_jars "$JARS"
-
-# Jars Gobblin runtime depends on
-# Please note that both versions of the metrics jar are required.
-function join { local IFS="$1"; shift; echo "$*"; }
-LIBJARS=(
- $USER_JARS
- $FWDIR_LIB/gobblin-api-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-avro-json-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-codecs-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-core-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-core-base-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-crypto-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-crypto-provider-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-data-management-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-metastore-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-metrics-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-metrics-base-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-metadata-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/gobblin-utility-$GOBBLIN_VERSION.jar
- $FWDIR_LIB/avro-1.8.1.jar
- $FWDIR_LIB/avro-mapred-1.8.1.jar
- $FWDIR_LIB/commons-lang3-3.4.jar
- $FWDIR_LIB/config-1.2.1.jar
- $FWDIR_LIB/data-11.0.0.jar
- $FWDIR_LIB/gson-2.6.2.jar
- $FWDIR_LIB/guava-15.0.jar
- $FWDIR_LIB/guava-retrying-2.0.0.jar
- $FWDIR_LIB/joda-time-2.9.3.jar
- $FWDIR_LIB/javassist-3.18.2-GA.jar
- $FWDIR_LIB/kafka_2.11-0.8.2.2.jar
- $FWDIR_LIB/kafka-clients-0.8.2.2.jar
- $FWDIR_LIB/metrics-core-2.2.0.jar
- $FWDIR_LIB/metrics-core-3.2.3.jar
- $FWDIR_LIB/metrics-graphite-3.2.3.jar
- $FWDIR_LIB/scala-library-2.11.8.jar
- $FWDIR_LIB/influxdb-java-2.1.jar
- $FWDIR_LIB/okhttp-2.4.0.jar
- $FWDIR_LIB/okio-1.4.0.jar
- $FWDIR_LIB/reactive-streams-1.0.0.jar
- $FWDIR_LIB/retrofit-1.9.0.jar
- $FWDIR_LIB/reflections-0.9.10.jar
-)
-LIBJARS=$(join , "${LIBJARS[@]}")
-
-# Add libraries to the Hadoop classpath
-GOBBLIN_DEP_JARS=`echo "$USER_JARS" | tr ',' ':' `
-for jarFile in `ls $FWDIR_LIB/*`
-do
- GOBBLIN_DEP_JARS=${GOBBLIN_DEP_JARS}:$jarFile
-done
-
-# Honor Gobblin dependencies
-export HADOOP_USER_CLASSPATH_FIRST=true
-export HADOOP_CLASSPATH=$GOBBLIN_DEP_JARS:$HADOOP_CLASSPATH
-
-GOBBLIN_CONFIG_FILE=$FWDIR_CONF/gobblin-mapreduce.properties
-
-JT_COMMAND=$([ -z $JOB_TRACKER_URL ] && echo "" || echo "-jt $JOB_TRACKER_URL")
-FS_COMMAND=$([ -z $FS_URL ] && echo "" || echo "-fs $FS_URL")
-
-export HADOOP_CLIENT_OPTS="$HADOOP_CLIENT_OPTS -Dgobblin.logs.dir=$GOBBLIN_LOG_DIR -Dlog4j.configuration=file:$FWDIR_CONF/log4j-mapreduce.xml"
-
-# Launch the job to run on Hadoop
-$HADOOP_BIN_DIR/hadoop jar \
- $FWDIR_LIB/gobblin-runtime-$GOBBLIN_VERSION.jar \
- org.apache.gobblin.runtime.mapreduce.CliMRJobLauncher \
- -D mapreduce.user.classpath.first=true \
- -D mapreduce.job.user.classpath.first=true \
- $JT_COMMAND \
- $FS_COMMAND \
- -libjars $LIBJARS \
- -sysconfig $GOBBLIN_CONFIG_FILE \
- -jobconfig $JOB_CONFIG_FILE
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin service mapreduce $@
\ No newline at end of file
diff --git a/bin/gobblin-service.sh b/bin/gobblin-service.sh
old mode 100644
new mode 100755
index 9df7b67..cafb15c
--- a/bin/gobblin-service.sh
+++ b/bin/gobblin-service.sh
@@ -17,123 +17,7 @@
# limitations under the License.
#
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
-function print_usage() {
- echo "gobblin-service.sh <start | stop>"
- echo "Where OPTION can be:"
- echo " --jvmflags <string of jvm flags> String containing any additional JVM flags to include"
- echo " --jars <column-separated list of extra jars> Column-separated list of extra jars to put on the CLASSPATH"
- echo " --help Display this help and exit"
-}
-
-function start() {
- for jarFile in `ls ${FWDIR_LIB}/*`
- do
- GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
- done
-
- export HADOOP_USER_CLASSPATH_FIRST=true
-
- CLASSPATH=${FWDIR_CONF}:${GOBBLIN_JARS}:${SERVICE_CONF_DIR}:${HADOOP_HOME}/lib
- if [ -n "$EXTRA_JARS" ]; then
- CLASSPATH=$CLASSPATH:"$EXTRA_JARS"
- fi
-
- LOG_ARGS="1>${FWDIR_LOGS}/GobblinService.stdout 2>${FWDIR_LOGS}/GobblinService.stderr"
-
-# LOG4J_ARGS="-Dlog4j.configuration=conf/log4j.xml"
- DEBUGGER_ARGS="-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=5006"
-
- LOG4J_PATH=file://${FWDIR_CONF}/log4j-service.properties
- COMMAND="$JAVA_HOME/bin/java -Dlog4j.configuration=$LOG4J_PATH -cp $CLASSPATH $JVM_FLAGS $LOG4J_ARGS $DEBUGGER_ARGS org.apache.gobblin.service.modules.core.GobblinServiceManager --service_name $SERVICE_NAME $LOG_ARGS"
-
- echo "Running command:"
- echo "$COMMAND"
- nohup $COMMAND >service.out 2>&1 & echo $! > $PID
-}
-
-function stop() {
- if [ -f "$PID" ]; then
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo 'Stopping the Gobblin service'
- kill $PID_VALUE
- else
- echo "Process $PID_VALUE is not running"
- fi
- else
- echo "No pid file found"
- fi
-}
-
-FWDIR="$(cd `dirname $0`/..; pwd)"
-FWDIR_LIB=${FWDIR}/lib
-FWDIR_CONF=${FWDIR}/conf/service
-FWDIR_BIN=${FWDIR}/bin
-FWDIR_LOGS=${FWDIR}/logs
-SERVICE_NAME="gobblin_service"
-
-. ${FWDIR_BIN}/gobblin-env.sh
-
-for i in "$@"
-do
- case "$1" in
- start|stop)
- ACTION="$1"
- ;;
- --jvmflags)
- JVM_FLAGS="$2"
- shift
- ;;
- --jars)
- EXTRA_JARS="$2"
- shift
- ;;
- --service)
- SERVICE_NAME="$2"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
-if [ -n "$JVM_FLAGS" ]; then
- JVM_FLAGS="-Xmx1g -Xms512m"
-fi
-
-PID="$FWDIR/.gobblin-service-app-pid"
-
-if [ -f "$PID" ]; then
- PID_VALUE=`cat $PID` > /dev/null 2>&1
-else
- PID_VALUE=""
-fi
-
-case "$ACTION" in
- "start")
- start
- ;;
- "stop")
- stop
- ;;
- *)
- print_usage
- exit 1
- ;;
-esac
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin service service-manager $@
\ No newline at end of file
diff --git a/bin/gobblin-standalone.sh b/bin/gobblin-standalone.sh
index 0dd66ca..356d3c4 100755
--- a/bin/gobblin-standalone.sh
+++ b/bin/gobblin-standalone.sh
@@ -17,248 +17,7 @@
# limitations under the License.
#
-function print_usage(){
- echo "gobblin-standalone.sh <start | status | restart | stop> [OPTION]"
- echo "Where OPTION can be:"
- echo " --workdir <job work dir> Gobblin's base work directory: if not set, taken from \${GOBBLIN_WORK_DIR}"
- echo " --fwdir <fwd dir> Gobblin's dist directory: if not set, taken from \${GOBBLIN_FWDIR}"
- echo " --logdir <log dir> Gobblin's log directory: if not set, taken from \${GOBBLIN_LOG_DIR}"
- echo " --jars <comma-separated list of job jars> Job jar(s): if not set, "$FWDIR_LIB" is examined"
- echo " --conf <directory of job configuration files> Directory of job configuration files: if not set, taken from ${GOBBLIN_JOB_CONFIG_DIR}"
- echo " --conffile <custom config file> Custom config file: if not set, is ignored. Overwrites properties in "$FWDIR_CONF/gobblin-standalone.properties
- echo " --jvmflags <string of jvm flags> String containing any additional JVM flags to include"
- echo " --help Display this help and exit"
-}
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
-
-for i in "$@"
-do
- case "$1" in
- start|stop|restart|status)
- ACTION="$1"
- ;;
- --workdir)
- WORK_DIR="$2"
- shift
- ;;
- --fwdir)
- FWDIR="$2"
- shift
- ;;
- --logdir)
- GOBBLIN_LOG_DIR="$2"
- shift
- ;;
- --jars)
- JARS="$2"
- shift
- ;;
- --conf)
- JOB_CONFIG_DIR="$2"
- shift
- ;;
- --conffile)
- CUSTOM_CONFIG_FILE="$2"
- shift
- ;;
- --jvmflags)
- JVM_FLAGS="$2"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-check=false
-if [ "$ACTION" == "start" ] || [ "$ACTION" == "restart" ]; then
- check=true
-fi
-
-if [ -n "$FWDIR" ]; then
- export GOBBLIN_FWDIR="$FWDIR"
-fi
-
-if [ -z "$GOBBLIN_FWDIR" ] && [ "$check" == true ]; then
- GOBBLIN_FWDIR="$(cd `dirname $0`/..; pwd)"
-fi
-
-FWDIR_LIB=$GOBBLIN_FWDIR/lib
-FWDIR_CONF=$GOBBLIN_FWDIR/conf
-
-# User defined job configuration directory overrides $GOBBLIN_JOB_CONFIG_DIR
-if [ -n "$JOB_CONFIG_DIR" ]; then
- export GOBBLIN_JOB_CONFIG_DIR="$JOB_CONFIG_DIR"
-fi
-
-if [ -z "$GOBBLIN_JOB_CONFIG_DIR" ] && [ "$check" == true ]; then
- die "Environment variable GOBBLIN_JOB_CONFIG_DIR not set!"
-fi
-
-# User defined work directory overrides $GOBBLIN_WORK_DIR
-if [ -n "$WORK_DIR" ]; then
- export GOBBLIN_WORK_DIR="$WORK_DIR"
-fi
-
-if [ -z "$GOBBLIN_WORK_DIR" ] && [ "$check" == true ]; then
- die "GOBBLIN_WORK_DIR is not set!"
-fi
-
-# User defined log directory overrides $GOBBLIN_LOG_DIR
-if [ -n "$LOG_DIR" ]; then
- export GOBBLIN_LOG_DIR="$LOG_DIR"
-fi
-
-if [ -z "$GOBBLIN_LOG_DIR" ] && [ "$check" == true ]; then
- GOBBLIN_LOG_DIR="$GOBBLIN_FWDIR/logs"
-fi
-
-# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
-if [ -n "$JVM_FLAGS" ]; then
- export GOBBLIN_JVM_FLAGS="$JVM_FLAGS"
-fi
-
-# User defined configuration file overrides $GOBBLIN_CUSTOM_CONFIG_FILE
-if [ -n "$CUSTOM_CONFIG_FILE" ]; then
- export GOBBLIN_CUSTOM_CONFIG_FILE="$CUSTOM_CONFIG_FILE"
-fi
-
-DEFAULT_CONFIG_FILE=$FWDIR_CONF/gobblin-standalone.properties
-
-PID="$GOBBLIN_WORK_DIR/.gobblin-pid"
-
-if [ -f "$PID" ]; then
- PID_VALUE=`cat $PID` > /dev/null 2>&1
-else
- PID_VALUE=""
-fi
-
-if [ ! -d "$GOBBLIN_LOG_DIR" ]; then
- mkdir "$GOBBLIN_LOG_DIR"
-fi
-
-set_user_jars(){
- local separator=''
- if [ -n "$1" ]; then
- IFS=','
- read -ra userjars <<< "$1"
- for userjar in ${userjars[@]}; do
- add_user_jar "$userjar"
- done
- unset IFS
- fi
-}
-
-add_user_jar(){
- local dirname=`dirname "$1"`
- local jarname=`basename "$1"`
- dirname=`cd "$dirname">/dev/null; pwd`
- GOBBLIN_JARS+="$separator$dirname/$jarname"
- separator=':'
-}
-
-# Add the absoulte path of the user defined job jars to the GOBBLIN_JARS
-set_user_jars "$JARS"
-
-start() {
- for jar in $(ls -d $FWDIR_LIB/*); do
- if [ "$GOBBLIN_JARS" != "" ]; then
- GOBBLIN_JARS+=":$jar"
- else
- GOBBLIN_JARS=$jar
- fi
- done
-
- CLASSPATH="$GOBBLIN_JARS:$FWDIR_CONF"
-
- echo "Starting Gobblin standalone daemon"
- COMMAND="$JAVA_HOME/bin/java -Xmx2g -Xms1g "
- COMMAND+="-XX:+UseConcMarkSweepGC -XX:+UseParNewGC "
- COMMAND+="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution "
- COMMAND+="-XX:+UseCompressedOops "
- COMMAND+="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$GOBBLIN_LOG_DIR/ "
- COMMAND+="-Xloggc:$GOBBLIN_LOG_DIR/gobblin-gc.log "
- COMMAND+="-Dgobblin.logs.dir=$GOBBLIN_LOG_DIR "
- COMMAND+="-Dlog4j.configuration=file://$FWDIR_CONF/log4j-standalone.xml "
- COMMAND+="-cp $CLASSPATH "
- COMMAND+="-Dorg.quartz.properties=$FWDIR_CONF/quartz.properties "
- COMMAND+="$GOBBLIN_JVM_FLAGS "
- COMMAND+="org.apache.gobblin.scheduler.SchedulerDaemon $DEFAULT_CONFIG_FILE $GOBBLIN_CUSTOM_CONFIG_FILE"
- echo "Running command:"
- echo "$COMMAND"
- nohup $COMMAND & echo $! > $PID
-}
-
-stop() {
- if [ -f "$PID" ]; then
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo 'Stopping Gobblin standalone daemon'
- kill $PID_VALUE
- sleep 1
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo "Gobblin standalone daemon did not stop gracefully, killing with kill -9"
- kill -9 $PID_VALUE
- echo "Cleaning up lock directory: $GOBBLIN_WORK_DIR/locks"
- rm -r $GOBBLIN_WORK_DIR/locks
- fi
- else
- echo "Process $PID_VALUE is not running"
- fi
- else
- echo "No pid file found"
- fi
-}
-
-# Check the status of the process
-status() {
- if [ -f "$PID" ]; then
- echo "Looking into file: $PID"
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo "Gobblin standalone daemon is running with status: "
- ps -ef | grep -v grep | grep $PID_VALUE
- else
- echo "Gobblin standalone daemon is not running"
- exit 1
- fi
- else
- echo "No pid file found"
- exit 1
- fi
-}
-
-case "$ACTION" in
- "start")
- start
- ;;
- "status")
- status
- ;;
- "restart")
- stop
- echo "Sleeping..."
- sleep 1
- start
- ;;
- "stop")
- stop
- ;;
- *)
- print_usage
- exit 1
- ;;
-esac
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin service standalone $@
\ No newline at end of file
diff --git a/bin/gobblin-yarn.sh b/bin/gobblin-yarn.sh
index 232e719..e531651 100755
--- a/bin/gobblin-yarn.sh
+++ b/bin/gobblin-yarn.sh
@@ -17,113 +17,7 @@
# limitations under the License.
#
-# Print an error message and exit
-function die() {
- echo -e "\nError: $@\n" 1>&2
- print_usage
- exit 1
-}
-
-function print_usage() {
- echo "gobblin-yarn.sh <start | stop>"
- echo "Where OPTION can be:"
- echo " --jvmflags <string of jvm flags> String containing any additional JVM flags to include"
- echo " --jars <column-separated list of extra jars> Column-separated list of extra jars to put on the CLASSPATH"
- echo " --help Display this help and exit"
-}
-
-function start() {
- for jarFile in `ls ${FWDIR_LIB}/*`
- do
- GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
- done
-
- export HADOOP_USER_CLASSPATH_FIRST=true
-
- CLASSPATH=${FWDIR_CONF}:${GOBBLIN_JARS}:${YARN_CONF_DIR}:${HADOOP_YARN_HOME}/lib
- if [ -n "$EXTRA_JARS" ]; then
- CLASSPATH=$CLASSPATH:"$EXTRA_JARS"
- fi
-
- COMMAND="$JAVA_HOME/bin/java -cp $CLASSPATH $JVM_FLAGS org.apache.gobblin.yarn.GobblinYarnAppLauncher"
-
- echo "Running command:"
- echo "$COMMAND"
- nohup $COMMAND & echo $! > $PID
-}
-
-function stop() {
- if [ -f "$PID" ]; then
- if kill -0 $PID_VALUE > /dev/null 2>&1; then
- echo 'Stopping the Gobblin Yarn application'
- kill $PID_VALUE
- else
- echo "Process $PID_VALUE is not running"
- fi
- else
- echo "No pid file found"
- fi
-}
-
-FWDIR="$(cd `dirname $0`/..; pwd)"
-FWDIR_LIB=${FWDIR}/lib
-FWDIR_CONF=${FWDIR}/conf/yarn
-FWDIR_BIN=${FWDIR}/bin
-
-. ${FWDIR_BIN}/gobblin-env.sh
-
-for i in "$@"
-do
- case "$1" in
- start|stop)
- ACTION="$1"
- ;;
- --jvmflags)
- JVM_FLAGS="$2"
- shift
- ;;
- --jars)
- EXTRA_JARS="$2"
- shift
- ;;
- --help)
- print_usage
- exit 0
- ;;
- *)
- ;;
- esac
- shift
-done
-
-if [ -z "$JAVA_HOME" ]; then
- die "Environment variable JAVA_HOME not set!"
-fi
-
-# User defined JVM flags overrides $GOBBLIN_JVM_FLAGS (if any)
-if [ -z "$JVM_FLAGS" ]; then
- JVM_FLAGS="-Xmx1g -Xms512m"
-fi
-
-PID="$FWDIR/.gobblin-yarn-app-pid"
-
-if [ -f "$PID" ]; then
- PID_VALUE=`cat $PID` > /dev/null 2>&1
-else
- PID_VALUE=""
-fi
-
-case "$ACTION" in
- "start")
- start
- ;;
- "stop")
- stop
- ;;
- *)
- print_usage
- exit 1
- ;;
-esac
-
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin service yarn $@
\ No newline at end of file
diff --git a/bin/gobblin.sh b/bin/gobblin.sh
index 5681f9c..d8d97fd 100755
--- a/bin/gobblin.sh
+++ b/bin/gobblin.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
@@ -17,50 +17,583 @@
# limitations under the License.
#
-calling_dir() {
- echo "$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+# JAVA_HOME is required.
+if [[ -z "$JAVA_HOME" ]]; then
+ echo -e "\nError: Environment variable JAVA_HOME not set!\n"
+ exit 1
+fi
+
+# global vars
+
+GOBBLIN_VERSION=@project.version@
+GOBBLIN_HOME="$(cd `dirname $0`/..; pwd)"
+GOBBLIN_LIB=${GOBBLIN_HOME}/lib
+GOBBLIN_BIN=${GOBBLIN_HOME}/bin
+GOBBLIN_LOGS=${GOBBLIN_HOME}/logs
+GOBBLIN_CONF=''
+
+#sourcing basic gobblin env vars like GOBBLIN_HOME and GOBBLIN_LIB
+. ${GOBBLIN_BIN}/gobblin-env.sh
+
+CLUSTER_NAME="gobblin_cluster"
+JVM_OPTS="-Xmx1g -Xms512m"
+LOG4J_FILE_PATH=''
+LOG4J_OPTS=''
+GOBBLIN_MODE=''
+ACTION=''
+JVM_FLAGS=''
+EXTRA_JARS=''
+VERBOSE=0
+ENABLE_GC_LOGS=0
+CMD_PARAMS=''
+
+# Gobblin Commands, Modes & respective Classes
+GOBBLIN_MODE_TYPE=''
+CLI='cli'
+SERVICE='service'
+
+# Commands
+CLASSPATH_CMD='classpath'
+
+# Execution Modes
+STANDALONE_MODE='standalone'
+CLUSTER_MASTER_MODE='cluster-master'
+CLUSTER_WORKER_MODE='cluster-worker'
+AWS_MODE='aws'
+YARN_MODE='yarn'
+MAPREDUCE_MODE='mapreduce'
+GOBBLIN_AS_SERVICE_MODE='gobblin-as-service'
+
+GOBBLIN_EXEC_MODE_LIST="$STANDALONE_MODE $CLUSTER_MASTER_MODE $CLUSTER_WORKER_MODE $AWS_MODE $YARN_MODE $MAPREDUCE_MODE $GOBBLIN_AS_SERVICE_MODE"
+
+# CLI Command class
+CLI_CLASS='org.apache.gobblin.runtime.cli.GobblinCli'
+
+# Service Class
+STANDALONE_CLASS='org.apache.gobblin.scheduler.SchedulerDaemon'
+CLUSTER_MASTER_CLASS='org.apache.gobblin.cluster.GobblinClusterManager'
+CLUSTER_WORKER_CLASS='org.apache.gobblin.cluster.GobblinTaskRunner'
+AWS_CLASS='org.apache.gobblin.aws.GobblinAWSClusterLauncher'
+YARN_CLASS='org.apache.gobblin.yarn.GobblinYarnAppLauncher'
+MAPREDUCE_CLASS='org.apache.gobblin.runtime.mapreduce.CliMRJobLauncher'
+SERVICE_MANAGER_CLASS='org.apache.gobblin.service.modules.core.GobblinServiceManager'
+
+function print_gobblin_usage() {
+ echo "Usage:"
+ echo "gobblin.sh cli <cli-command> <params>"
+ echo "gobblin.sh service <execution-mode> <start|stop|restart|status>"
+ echo ""
+ echo "Use \"gobblin <cli|service> --help\" for more information. (Gobblin Version: $GOBBLIN_VERSION)"
+}
+
+function print_gobblin_cli_usage() {
+ echo "Usage:"
+ echo "gobblin.sh cli <cli-command> <params>"
+ echo ""
+ echo "options:"
+ echo " cli-commands:
+ passwordManager Encrypt or decrypt strings for the password manager.
+ decrypt Decryption utilities
+ run Run a Gobblin application.
+ config Query the config library
+ jobs Command line job info and operations
+ stateMigration Command line tools for migrating state store
+ job-state-to-json To convert Job state to JSON
+ cleaner Data retention utility
+ keystore Examine JCE Keystore files
+ watermarks Inspect streaming watermarks
+ job-store-schema-manager Database job history store schema manager
+ gobblin-classpath shows the constructed gobblin classpath"
+ echo ""
+ echo " --conf-dir <gobblin-conf-dir-path> Gobblon config path. default is '\$GOBBLIN_HOME/conf/<exe-mode-name>'."
+ echo " --log4j-conf <path-of-log4j-file> default is '<gobblin-conf-dir-path>/<execution-mode>/log4j.properties'."
+ echo " --jvmopts <jvm or gc options> String containing JVM flags to include, in addition to \"$JVM_OPTS\"."
+ echo " --jars <csv list of extra jars> Column-separated list of extra jars to put on the CLASSPATH."
+ echo " --enable-gc-logs enables gc logs & dumps."
+ echo " --show-classpath prints gobblin runtime classpath."
+ echo " --help Display this help."
+ echo " --verbose Display full command used to start the process."
+ echo " Gobblin Version: $GOBBLIN_VERSION"
+}
+
+function print_gobblin_service_usage() {
+ echo "Usage:"
+ echo "gobblin.sh service <execution-mode> <start|stop|status>"
+ echo ""
+ echo "Argument Options:"
+ echo " <execution-mode> $GOBBLIN_EXEC_MODE_LIST."
+ echo ""
+ echo " --conf-dir <gobblin-conf-dir-path> Gobblin config path. default is '\$GOBBLIN_HOME/conf/<exe-mode-name>'."
+ echo " --log4j-conf <path-of-log4j-file> default is '<gobblin-conf-dir-path>/<execution-mode>/log4j.properties'."
+ echo " --jvmopts <jvm or gc options> String containing JVM flags to include, in addition to \"$JVM_OPTS\"."
+ echo " --jars <csv list of extra jars> Column-separated list of extra jars to put on the CLASSPATH."
+ echo " --enable-gc-logs enables gc logs & dumps."
+ echo " --show-classpath prints gobblin runtime classpath."
+ echo " --cluster-name Name of the cluster to be used by helix & other services. ( default: $CLUSTER_NAME)."
+ echo " --jt <resource manager URL> Only for mapreduce mode: Job submission URL, if not set, taken from \${HADOOP_HOME}/conf."
+ echo " --fs <file system URL> Only for mapreduce mode: Target file system, if not set, taken from \${HADOOP_HOME}/conf."
+ echo " --job-conf-file <job-conf-file-path> Only for mapreduce mode: configuration file for the job to run"
+ echo " --help Display this help."
+ echo " --verbose Display full command used to start the process."
+ echo " Gobblin Version: $GOBBLIN_VERSION"
}
-classpath() {
- DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
-
- for i in `ls $DIR/../lib`
- do
- if [[ $i != hadoop* ]]
- then
- CLASSPATH=${CLASSPATH:+${CLASSPATH}:}$DIR/../lib/$i
- else
- HADOOP_CLASSPATH=${HADOOP_CLASSPATH:+${HADOOP_CLASSPATH}:}$DIR/../lib/$i
- fi
- done
-
- if [ ! -z "$HADOOP_HOME" ] && [ -f $HADOOP_HOME/bin/hadoop ]
- then
- HADOOP_CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath)
- fi
-
- CLASSPATH=$CLASSPATH:$HADOOP_CLASSPATH
-
- if [ ! -z "$GOBBLIN_ADDITIONAL_JARS" ]
- then
- CLASSPATH=$GOBBLIN_ADDITIONAL_JARS:$CLASSPATH
- fi
-
- echo $CLASSPATH
+
+function print_help_n_exit() {
+ if [[ "$GOBBLIN_MODE_TYPE" == "$CLI" ]]; then
+ print_gobblin_cli_usage
+ elif [[ "$GOBBLIN_MODE_TYPE" == "$SERVICE" ]]; then
+ print_gobblin_service_usage
+ else
+ print_gobblin_usage
+ fi
+ exit 1;
}
+# TODO: use getopts
+shopt -s nocasematch
for i in "$@"
do
- case "$1" in
- "classpath")
- classpath
- exit
- esac
+ case "$1" in
+ "$CLI" )
+ GOBBLIN_MODE_TYPE=$CLI
+ GOBBLIN_MODE="$1"
+ ;;
+
+ "$SERVICE" )
+ GOBBLIN_MODE_TYPE=$SERVICE
+ if [[ " $GOBBLIN_EXEC_MODE_LIST " =~ .*\ $2\ .* ]]; then
+ GOBBLIN_MODE="$2"
+ shift
+ else
+ echo "ERROR: Service $2 is not supported. Valid services are : $GOBBLIN_EXEC_MODE_LIST"
+ print_help_n_exit
+ fi
+ ;;
+
+ start | stop | restart | status)
+ ACTION="$1"
+ ;;
+
+ --show-classpath)
+ CMD_PARAMS='dummy'
+ GOBBLIN_MODE=$CLI
+ GOBBLIN_MODE_TYPE=$CLI
+ GOBBLIN_MODE=$CLASSPATH_CMD
+ ;;
+ --jvmflags)
+ JVM_FLAGS="$2"
+ shift
+ ;;
+ --conf-dir)
+ USER_CONF_DIR="$2"
+ shift
+ ;;
+ --log4j-conf)
+ USER_LOG4J_FILE="$2"
+ shift
+ ;;
+ --jars)
+ EXTRA_JARS="$2"
+ shift
+ ;;
+ --enable-gc-logs)
+ ENABLE_GC_LOGS=1
+ ;;
+ --cluster-name)
+ CLUSTER_NAME="$2"
+ shift
+ ;;
+ --help )
+ print_help_n_exit
+ ;;
+ --verbose)
+ VERBOSE=1
+ ;;
+ --jt)
+ JOB_TRACKER_URL="$2"
+ shift
+ ;;
+ --fs)
+ FS_URL="$2"
+ shift
+ ;;
+ --job-conf-file)
+ JOB_CONF_FILE="$2"
+ shift
+ ;;
+ *)
+ CMD_PARAMS="$CMD_PARAMS $1"
+ ;;
+ esac
+ shift
done
-CLASSPATH=$(classpath)
-if [ -z "$GOBBLIN_LOG4J_CONFIGURATION" ]
-then
- GOBBLIN_LOG4J_CONFIGURATION=$(calling_dir)/../conf/log4j.properties
+# for gobblin commands, the action is always 'start'
+if [[ "$GOBBLIN_MODE_TYPE" == "$CLI" ]]; then
+ ACTION='start'
+ # print help by default if any of the supported command is not specified
+ if [[ -z "$GOBBLIN_MODE" || -z "$CMD_PARAMS" ]]; then
+ echo "ERROR: command parameters are required for $GOBBLIN_MODE"
+ print_help_n_exit
+ fi
fi
-java -Dlog4j.configuration=file:$GOBBLIN_LOG4J_CONFIGURATION -cp "$CLASSPATH" $GOBBLIN_OPTS org.apache.gobblin.runtime.cli.GobblinCli $@
+# derived based on input from user, $GOBBLIN_MODE
+PID_FILE_NAME=".gobblin-$GOBBLIN_MODE.pid"
+PID_FILE="$GOBBLIN_HOME/$PID_FILE_NAME"
+
+
+# JVM Flags
+if [[ -n "$JVM_FLAGS" ]]; then
+ JVM_OPTS="$JVM_OPTS $JVM_FLAGS"
+fi
+
+# gobblin config
+if [[ -n "$USER_CONF_DIR" ]]; then
+ GOBBLIN_CONF=$USER_CONF_DIR
+else
+ GOBBLIN_CONF=${GOBBLIN_HOME}/conf/${GOBBLIN_MODE}
+fi
+
+#log4j config file
+if [[ -n "$USER_LOG4J_FILE" ]]; then
+ LOG4J_FILE_PATH=file://${USER_LOG4J_FILE}
+ LOG4J_OPTS="-Dlog4j.configuration=$LOG4J_FILE_PATH"
+#preference to log4j2.xml for log4j2 support
+elif [[ -f ${GOBBLIN_CONF}/log4j2.xml ]]; then
+ LOG4J_FILE_PATH=file://${GOBBLIN_CONF}/log4j2.xml
+ LOG4J_OPTS="-Dlog4j.configuration=$LOG4J_FILE_PATH"
+#defaults to log4j.properties
+elif [[ -f ${GOBBLIN_CONF}/log4j.properties ]]; then
+ LOG4J_FILE_PATH=file://${GOBBLIN_CONF}/log4j.properties
+ LOG4J_OPTS="-Dlog4j.configuration=$LOG4J_FILE_PATH"
+fi
+
+if [[ ! -d "$GOBBLIN_LOGS" ]]; then
+ mkdir -p $GOBBLIN_LOGS
+fi
+
+GC_OPTS=''
+if [[ ${ENABLE_GC_LOGS} -eq 1 ]]; then
+ GC_OPTS+="-XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseCompressedOops "
+ GC_OPTS+="-XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution "
+ GC_OPTS+="-XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$GOBBLIN_LOGS/ "
+ GC_OPTS+="-Xloggc:$GOBBLIN_LOGS/gobblin-$GOBBLIN_MODE-gc.log "
+fi
+
+
+function build_classpath(){
+ GOBBLIN_CLASSPATH=''
+ # Build classpth
+ GOBBLIN_JARS=''
+ GOBBLIN_HADOOP_JARS=''
+ GOBBLIN_CLASSPATH=''
+ GOBBLIN_HADOOP_CLASSPATH=''
+
+ for jarFile in ${GOBBLIN_LIB}/*.jar
+ do
+ [[ -e "$jarFile" ]] || break # in case there is no jar file
+ if [[ $jarFile == hadoop* ]]; then
+ GOBBLIN_HADOOP_JARS=${GOBBLIN_HADOOP_JARS}:${jarFile}
+ else
+ GOBBLIN_JARS=${GOBBLIN_JARS}:${jarFile}
+ fi
+ done
+
+ # just removing first colon if present
+ GOBBLIN_HADOOP_JARS=${GOBBLIN_HADOOP_JARS#:}
+ GOBBLIN_JARS=${GOBBLIN_JARS#:}
+
+ if [[ -n "$HADOOP_HOME" ]]; then
+ GOBBLIN_HADOOP_CLASSPATH=$($HADOOP_HOME/bin/hadoop classpath)
+ GOBBLIN_CLASSPATH=${GOBBLIN_JARS}:${GOBBLIN_HADOOP_CLASSPATH}
+ else
+ echo "WARN: HADOOP_HOME is not defined. Gobblin Hadoop libs will be used in classpath."
+ GOBBLIN_HADOOP_CLASSPATH=${GOBBLIN_HADOOP_JARS}
+ fi
+
+ GOBBLIN_CLASSPATH=${GOBBLIN_JARS}:${GOBBLIN_HADOOP_CLASSPATH}
+
+ if [[ -n "$EXTRA_JARS" ]]; then
+ GOBBLIN_CLASSPATH=${GOBBLIN_CLASSPATH}:"$EXTRA_JARS"
+ fi
+
+ GOBBLIN_CLASSPATH=${GOBBLIN_CONF}:${GOBBLIN_CLASSPATH}
+
+}
+
+function get_gobblin_mapreduce_libs() {
+
+ if [[ -z "$GOBBLIN_VERSION" ]]; then
+ echo ""
+ return
+ fi
+
+ GOBBLIN_MR_JARS=(
+ commons-cli-1.3.1.jar
+ mysql-connector-java-*.jar
+ avro-*.jar
+ commons-lang3-*.jar
+ config-*.jar
+ data-*.jar
+ gson-*.jar
+ guava-*.jar
+ joda-time-*.jar
+ javassist-*.jar
+ kafka_2.11-*.jar
+ kafka-clients-0.8.2.2.jar
+ metrics-core-*.jar
+ metrics-graphite-*.jar
+ scala-library-*.jar
+ influxdb-java-*.jar
+ okhttp-*.jar
+ okio-*.jar
+ reactive-streams-*.jar
+ retrofit-*.jar
+ reflections-*.jar
+ gobblin-*.jar
+ )
+
+ old_IFS=$IFS
+ IFS=' '
+ MR_JARS=''
+ for i in "${GOBBLIN_MR_JARS[@]}"; do
+ for j in $(eval echo -e "$GOBBLIN_LIB/${i}"); do
+ MR_JARS="$MR_JARS,$j"
+ done
+ done
+ echo ${MR_JARS:1}
+ IFS=$old_IFS
+}
+
+function start() {
+ # build and set the classpath in variable: GOBBLIN_CLASSPATH
+ build_classpath
+
+ LOG_OUT_FILE="${GOBBLIN_LOGS}/${GOBBLIN_MODE}.out"
+ LOG_ERR_FILE="${GOBBLIN_LOGS}/${GOBBLIN_MODE}.err"
+
+ # for all gobblin commands
+ if [[ "$GOBBLIN_MODE_TYPE" == "$CLI" ]]; then
+ if [[ "$GOBBLIN_MODE" = "$CLASSPATH_CMD" ]]; then
+ # not adding anything in echo here so that it can be used for getting classpath from any other command
+ echo "$GOBBLIN_CLASSPATH"
+ else
+ #prints the command
+ if [[ $VERBOSE -eq 1 ]]; then
+ echo "Running command: $JAVA_HOME/bin/java $GC_OPTS $JVM_OPTS -cp $GOBBLIN_CLASSPATH $CLI_CLASS $CMD_PARAMS";
+ fi
+
+ # execute the command
+ $JAVA_HOME/bin/java $GC_OPTS $JVM_OPTS -cp $GOBBLIN_CLASSPATH $CLI_CLASS $CMD_PARAMS
+ fi
+ # for all gobblin execution modes
+ else
+ if [[ "$GOBBLIN_MODE" = "$MAPREDUCE_MODE" ]]; then
+ if [[ -z "$JOB_CONF_FILE" ]]; then
+ echo "--job-conf-file is required that specifies the job to be run into mapreduce mode by Gobblin."
+ exit 1;
+ fi
+
+ MR_MODE_LIB_JARS=$(get_gobblin_mapreduce_libs)
+ export HADOOP_CLASSPATH=${GOBBLIN_CLASSPATH}
+ export HADOOP_USER_CLASSPATH_FIRST=true
+ if [[ -n "$JVM_OPTS" ]]; then
+ export HADOOP_OPTS="$HADOOP_OPTS $JVM_OPTS"
+ fi
+
+ if [[ -n "$EXTRA_JARS" ]]; then
+ MR_MODE_LIB_JARS="$EXTRA_JARS,${MR_MODE_LIB_JARS}"
+ fi
+
+ JT_COMMAND=$([ -z $JOB_TRACKER_URL ] && echo "" || echo "-jt $JOB_TRACKER_URL")
+ FS_COMMAND=$([ -z $FS_URL ] && echo "" || echo "-fs $FS_URL")
+
+ GOBBLIN_COMMAND="hadoop jar $GOBBLIN_LIB/gobblin-runtime-$GOBBLIN_VERSION.jar $MAPREDUCE_CLASS \
+ -D mapreduce.job.user.classpath.first=true $JT_COMMAND $FS_COMMAND \
+ -libjars $MR_MODE_LIB_JARS -sysconfig $GOBBLIN_CONF/application.conf -jobconfig $JOB_CONF_FILE"
+
+ else
+ CLASS_N_ARGS=''
+ if [[ "$GOBBLIN_MODE" = "$STANDALONE_MODE" ]]; then
+ CLASS_N_ARGS="$STANDALONE_CLASS $GOBBLIN_CONF/application.conf"
+
+ elif [[ "$GOBBLIN_MODE" = "$AWS_MODE" ]]; then
+ CLASS_N_ARGS="$AWS_CLASS"
+
+ elif [[ "$GOBBLIN_MODE" = "$YARN_MODE" ]]; then
+ GOBBLIN_CLASSPATH="${GOBBLIN_CLASSPATH}:${HADOOP_YARN_HOME}/lib"
+ CLASS_N_ARGS="$YARN_CLASS"
+
+ elif [[ "$GOBBLIN_MODE" = "$CLUSTER_MASTER_MODE" ]]; then
+ CLASS_N_ARGS="$CLUSTER_MASTER_CLASS --standalone_cluster true --app_name $CLUSTER_NAME "
+
+ elif [[ "$GOBBLIN_MODE" = "$GOBBLIN_AS_SERVICE_MODE" ]]; then
+ CLASS_N_ARGS="$SERVICE_MANAGER_CLASS --service_name Gobblin-$GOBBLIN_AS_SERVICE_MODE --service_id GAAS-1"
+
+ elif [[ "$GOBBLIN_MODE" = "$CLUSTER_WORKER_MODE" ]]; then
+ #Find largest worker id and use next one to start worker in incremental order, starts with 1
+ LAST_WORKER_ID=$(ps aux | grep -v grep | grep -Po "($CLUSTER_WORKER_CLASS)(.*)(cluster-worker.\K([0-9]+))" | sort --version-sort | tail -1)
+ WORKER_ID=$((LAST_WORKER_ID+1))
+ LOG_OUT_FILE="${GOBBLIN_LOGS}/${GOBBLIN_MODE}.$WORKER_ID.out"
+ LOG_ERR_FILE="${GOBBLIN_LOGS}/${GOBBLIN_MODE}.$WORKER_ID.err"
+ CLASS_N_ARGS="$CLUSTER_WORKER_CLASS --app_name $CLUSTER_NAME --helix_instance_name ${GOBBLIN_MODE}.$WORKER_ID "
+ else
+ echo "Invalid gobblin command or execution mode... [EXITING]"
+ exit 1
+ fi
+ GOBBLIN_COMMAND="$JAVA_HOME/bin/java -cp $GOBBLIN_CLASSPATH $GC_OPTS $JVM_OPTS $LOG4J_OPTS $CLASS_N_ARGS"
+ fi
+
+ # execute the command
+ if [[ $VERBOSE -eq 1 ]]; then
+ echo "Running command: $GOBBLIN_COMMAND 1>> ${LOG_OUT_FILE} 2>> ${LOG_ERR_FILE}";
+ fi
+ nohup $GOBBLIN_COMMAND 1>> ${LOG_OUT_FILE} 2>> ${LOG_ERR_FILE} &
+ PID=$!
+ echo $PID >> $PID_FILE
+ if [[ $? != 0 ]]; then
+ echo "Starting the Gobblin $GOBBLIN_MODE process... [FAILED]"
+ else
+ echo "Started the Gobblin $GOBBLIN_MODE process [pid: $PID] ... [DONE]"
+ fi
+ fi
+}
+
+
+
+function stop() {
+ # echo "Stopping the Gobblin $MODE_TYPE process..."
+ PID=''
+ if [[ ! -f $PID_FILE ]]; then
+ echo "Gobblin process id file not found at $PID_FILE"
+ else
+ PID=`tail -1 $PID_FILE`
+ fi
+
+ if [[ -z "$PID" ]]; then
+ echo "Can not find any running Gobblin $GOBBLIN_MODE process..."
+ while true; do
+ read -p "Do you want to search gobblin $GOBBLIN_MODE process and stop it? (y/n): " search_and_kill
+ case ${search_and_kill} in
+ [Yy]*)
+ class_to_search=''
+ if [[ "$GOBBLIN_MODE" = "$MAPREDUCE_MODE" ]]; then
+ class_to_search="$MAPREDUCE_CLASS"
+ elif [[ "$GOBBLIN_MODE" = "$CLI" ]]; then
+ class_to_search="$CLI_CLASS"
+ elif [[ "$GOBBLIN_MODE" = "$STANDALONE_MODE" ]]; then
+ class_to_search="$STANDALONE_CLASS"
+ elif [[ "$GOBBLIN_MODE" = "$AWS_MODE" ]]; then
+ class_to_search="$AWS_CLASS"
+ elif [[ "$GOBBLIN_MODE" = "$YARN_MODE" ]]; then
+ class_to_search="$YARN_CLASS"
+ elif [[ "$GOBBLIN_MODE" = "$GOBBLIN_AS_SERVICE_MODE" ]]; then
+ class_to_search="$SERVICE_MANAGER_CLASS"
+ elif [[ "$GOBBLIN_MODE" = "$CLUSTER_MASTER_MODE" ]]; then
+ class_to_search="$CLUSTER_MASTER_CLASS"
+ elif [[ "$GOBBLIN_MODE" = "$CLUSTER_WORKER_MODE" ]]; then
+ class_to_search=$CLUSTER_WORKER_CLASS
+ fi
+
+ if [[ -z "$class_to_search" ]]; then
+ echo "Could not figure out process to search for MODE: $GOBBLIN_MODE...[ABORTED]"
+ exit
+ fi
+
+ PID=$(ps aux | grep "$class_to_search" | grep -v grep | awk '{print $2}')
+ break
+ ;;
+ [Nn]*)
+ echo "Stopping the Gobblin $GOBBLIN_MODE process... [ABORTED]"
+ exit
+ ;;
+ *)
+ echo "Please answer yes or no."
+ ;;
+ esac
+ done
+ fi
+
+ if [[ -n "$PID" ]]; then
+ if kill -0 $PID > /dev/null 2>&1; then
+ kill $PID
+ printf "Stopping the Gobblin $GOBBLIN_MODE process (pid: $PID)... "; sleep 1; printf "[DONE]\n"
+ else
+ echo "Gobblin $GOBBLIN_MODE process (pid: $PID) is not running."
+ fi
+ # remove the pid from pid_file, and remove the file if no more pid's left.
+ case "$(uname -s)" in
+ Darwin)
+ sed -i '' '$d' $PID_FILE
+ ;;
+ Linux)
+ sed -i '$d' $PID_FILE
+ ;;
+ CYGWIN*|MINGW32*|MSYS*)
+ # TODO: check if this command works on windows
+ sed -i '' '$d' $PID_FILE
+ ;;
+ *)
+ # do nothing
+ ;;
+ esac
+ if [[ -s $PID_FILE ]]; then
+ rm $PID_FILE;
+ fi
+ else
+ echo "Can not find any running Gobblin $GOBBLIN_MODE process...[FAILED]"
+ fi
+}
+
+function status_check() {
+ # echo "Checking for Gobblin $MODE_TYPE status ..."
+ if [[ -e ${PID_FILE} ]]; then
+ PID=`cat $PID_FILE`
+ fi
+ if [[ -z ${PID} ]]; then
+ echo "Gobblin $GOBBLIN_MODE process id not found, probably it is not running."
+ elif ps -p ${PID} > /dev/null; then
+ echo "Gobblin $GOBBLIN_MODE process is running... [ pid: $PID ]."
+ else
+ echo "Gobblin $GOBBLIN_MODE process is not running. [ last known pid: ${PID}]."
+ fi
+}
+
+
+function status() {
+ # echo "Checking for Gobblin $MODE_TYPE status ..."
+ if [[ -e ${PID_FILE} ]]; then
+ PID=`cat $PID_FILE`
+ fi
+ if [[ -z ${PID} ]]; then
+ echo "Gobblin $GOBBLIN_MODE process id not found, probably it is not running."
+ elif ps -p ${PID} > /dev/null; then
+ echo "Gobblin $GOBBLIN_MODE process is running... [ pid: $PID ]."
+ else
+ echo "Gobblin $GOBBLIN_MODE process is not running. [ last known pid: ${PID}]."
+ fi
+}
+
+
+case "$ACTION" in
+ "start")
+ start
+ ;;
+ "stop")
+ stop
+ ;;
+ "status")
+ status
+ ;;
+ "restart")
+ stop
+ sleep 3
+ start
+ ;;
+ *)
+ echo "ERROR: One of the action is required: start/stop/status/restart"
+ print_help_n_exit
+ ;;
+esac
+
diff --git a/conf/quartz.properties b/bin/gobblin_password_encryptor.sh
old mode 100644
new mode 100755
similarity index 75%
rename from conf/quartz.properties
rename to bin/gobblin_password_encryptor.sh
index 0ea5f30..0fdc211
--- a/conf/quartz.properties
+++ b/bin/gobblin_password_encryptor.sh
@@ -1,3 +1,5 @@
+#!/bin/bash
+
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -15,6 +17,9 @@
# limitations under the License.
#
-org.quartz.scheduler.instanceName = LocalJobScheduler
-org.quartz.threadPool.threadCount = 3
-org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
+
+script_dir=$(dirname $0)
+lib_dir=${script_dir}/../lib
+
+java -cp ${lib_dir}'/*' org.apache.gobblin.util.CLIPasswordEncryptor "$@"
diff --git a/bin/historystore-manager.sh b/bin/historystore-manager.sh
index 8938bef..c2fbc25 100755
--- a/bin/historystore-manager.sh
+++ b/bin/historystore-manager.sh
@@ -17,44 +17,7 @@
# limitations under the License.
#
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
-FWDIR="$(cd `dirname $0`/..; pwd)"
-
-GOBBLIN_JARS=""
-for jar in $(ls -d $FWDIR/lib/*); do
- if [ "$GOBBLIN_JARS" != "" ]; then
- GOBBLIN_JARS+=":$jar"
- else
- GOBBLIN_JARS=$jar
- fi
-done
-
-CLASSPATH=$GOBBLIN_JARS
-CLASSPATH+=":$FWDIR/conf"
-
-if [[ $# -gt 0 ]]; then
- action=$1
- options=( "$@" );
- args=( "$@" );
- unset options[$1];
- unset args[$1];
-fi
-
-index=1
-for var in "${options[@]}"
-do
- if [[ $var == -D=* ]]; then
- unset options[$index];
- fi
- index=$((index+1))
-done
-index=1
-for var in "${args[@]}"
-do
- if [[ $var != -D=* ]]; then
- unset args[$index];
- fi
- index=$((index+1))
-done
-
-java "${options[@]}" -cp $CLASSPATH org.apache.gobblin.metastore.util.DatabaseJobHistoryStoreSchemaManager $action "${args[@]}"
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin cli job-store-schema-manager $@
\ No newline at end of file
diff --git a/bin/statestore-checker.sh b/bin/statestore-checker.sh
index dbd5652..ff9661e 100755
--- a/bin/statestore-checker.sh
+++ b/bin/statestore-checker.sh
@@ -17,18 +17,7 @@
# limitations under the License.
#
-FWDIR="$(cd `dirname $0`/..; pwd)"
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
-GOBBLIN_JARS=""
-for jar in $(ls -d $FWDIR/lib/*); do
- if [ "$GOBBLIN_JARS" != "" ]; then
- GOBBLIN_JARS+=":$jar"
- else
- GOBBLIN_JARS=$jar
- fi
-done
-
-CLASSPATH=$GOBBLIN_JARS
-CLASSPATH+=":$FWDIR/conf"
-
-java -cp $CLASSPATH org.apache.gobblin.runtime.util.JobStateToJsonConverter $@
+CURRENT_DIR="$(cd `dirname $0`/..; pwd)"
+$CURRENT_DIR/bin/gobblin cli job-state-to-json $@
diff --git a/bin/statestore-cleaner.sh b/bin/statestore-cleaner.sh
index a5ddb7e..2999220 100755
--- a/bin/statestore-cleaner.sh
+++ b/bin/statestore-cleaner.sh
@@ -17,6 +17,8 @@
# limitations under the License.
#
+# @depricated: This script is kept for backward compatibility only and will be removed in future. Use gobblin.sh
+
FWDIR="$(cd `dirname $0`/..; pwd)"
GOBBLIN_JARS=""
diff --git a/conf/aws/application.conf b/conf/aws/application.conf
index be4e04c..33b5570 100644
--- a/conf/aws/application.conf
+++ b/conf/aws/application.conf
@@ -46,7 +46,7 @@ gobblin.aws.job.conf.s3.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/job
# Specify S3 URI for cluster configuration
# Steps to obtain this URI:
# 1. Create a S3 folder in a bucket, say the bucket name is "s3bucket" and folder is "/gobblin-confs/cluster-conf/"
-# 2. After editing this application.conf: upload this application.conf, log4j-aws.properties and quartz.properties to
+# 2. After editing this application.conf: upload this application.conf, log4j.properties and quartz.properties to
# /s3bucket/gobblin-confs/cluster-conf/
# Note: Make sure to replace AWS credential values with dummy values.
# We don't read AWS credentials from application.conf in S3
@@ -107,7 +107,7 @@ gobblin.aws.desired.workers=2
## Various directories
gobblin.aws.conf.dir=${gobblin.aws.nfs.root.dir}"/cluster-conf"
-gobblin.aws.master.conf.local=${gobblin.aws.conf.dir}"/log4j-aws.properties,"${gobblin.aws.conf.dir}"/application.conf"
+gobblin.aws.master.conf.local=${gobblin.aws.conf.dir}"/log4j.properties,"${gobblin.aws.conf.dir}"/application.conf"
gobblin.aws.worker.conf.local=${gobblin.aws.master.conf.local}
# Lib jars location is tied closely to AMI being used. Only change if you know what you are doing
gobblin.aws.lib.jars.dir="/home/ec2-user/gobblin-lib"
@@ -153,11 +153,11 @@ metrics.enabled=true
metrics.reporting.custom.builders=org.apache.gobblin.metrics.ConsoleEventReporterFactory
## All Gobblin Jars and Configuration on S3
-gobblin.aws.master.s3.conf.files="application.conf,log4j-aws.properties,quartz.properties"
+gobblin.aws.master.s3.conf.files="application.conf,log4j.properties,quartz.properties"
gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/latest-jars/"
# Note: change gobblin-version in gobblin jar names if you intend to run any sother gobblin version
## All Gobblin Jars and Configuration on S3
-gobblin.aws.master.s3.conf.files="application.conf,log4j-aws.properties,quartz.properties"
+gobblin.aws.master.s3.conf.files="application.conf,log4j.properties,quartz.properties"
gobblin.aws.master.s3.jars.uri="https://s3-us-west-2.amazonaws.com/gobblin-libs/latest-jars/"
gobblin.aws.master.s3.jars.files="ST4-4.0.4.jar,activation-1.1.1.jar,annotations-2.0.1.jar,ant-1.9.1.jar,ant-launcher-1.9.1.jar,antlr-runtime-3.5.2.jar,aopalliance-1.0.jar,apache-log4j-extras-1.2.17.jar,asm-3.1.jar,asm-commons-3.1.jar,asm-tree-3.1.jar,avro-1.7.7.jar,avro-ipc-1.7.7-tests.jar,avro-ipc-1.7.7.jar,avro-mapred-1.7.7-hadoop2.jar,aws-java-sdk-applicationautoscaling-1.11.8.jar,aws-java-sdk-autoscaling-1.11.8.jar,aws-java-sdk-core-1.11.8.jar,aws-java-sdk-ec2-1.11.8.jar,aws-java-sd [...]
diff --git a/conf/yarn/log4j-yarn.properties b/conf/aws/log4j.properties
similarity index 100%
rename from conf/yarn/log4j-yarn.properties
rename to conf/aws/log4j.properties
diff --git a/conf/gobblin-cli.properties b/conf/cli/gobblin-cli.properties
similarity index 100%
rename from conf/gobblin-cli.properties
rename to conf/cli/gobblin-cli.properties
diff --git a/conf/standalone/master/log4j-cluster.properties b/conf/cli/log4j.properties
similarity index 92%
copy from conf/standalone/master/log4j-cluster.properties
copy to conf/cli/log4j.properties
index 6effc68..2fb53c8 100755
--- a/conf/standalone/master/log4j-cluster.properties
+++ b/conf/cli/log4j.properties
@@ -19,6 +19,8 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n
# Suppressed loggers
+log4j.logger.org.reflections.Reflections=ERROR
+log4j.logger.org.apache.helix.controller.pipeline.Pipeline=ERROR
log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR
log4j.logger.org.apache.helix.controller.stages=ERROR
log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR
diff --git a/conf/standalone/worker/application.conf b/conf/cluster-master/application.conf
similarity index 78%
rename from conf/standalone/worker/application.conf
rename to conf/cluster-master/application.conf
index e9b8323..8b1d88e 100644
--- a/conf/standalone/worker/application.conf
+++ b/conf/cluster-master/application.conf
@@ -15,6 +15,18 @@
# limitations under the License.
#
+# Cluster configuration properties
+gobblin.cluster.app.name=GobblinStandaloneCluster
+gobblin.cluster.email.notification.on.shutdown=false
+gobblin.cluster.helix.instance.max.retries=2
+gobblin.cluster.work.dir=/tmp/gobblin-cluster
+
+# Helix/Zookeeper configuration properties
+gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
+gobblin.cluster.zk.connection.string="localhost:2181"
+
+jobconf.monitor.interval=30000
+
# Sample configuration properties for the Gobblin Standalone cluster
gobblin.cluster.workDir=${gobblin.cluster.work.dir}/GobblinStandaloneCluster
@@ -32,8 +44,10 @@ gobblin.cluster.helix.cluster.name=GobblinStandaloneClusterCli
gobblin.cluster.job.conf.path=${gobblin.cluster.work.dir}/jobs
gobblin.cluster.jobconf.fullyQualifiedPath=${gobblin.cluster.work.dir}/jobs
+gobblin.cluster.job.catalog=org.apache.gobblin.runtime.job_catalog.FSJobCatalog
# File system URIs
+fs.uri="file:///"
writer.fs.uri=${fs.uri}
state.store.fs.uri=${fs.uri}
@@ -69,3 +83,16 @@ task.status.reportintervalinms=1000
# Enable metrics / events
metrics.enabled=true
+# UI
+admin.server.enabled=true
+admin.server.port=9000
+
+# is this required/redundent ?
+rest.server.host=localhost
+rest.server.port=9090
+
+# job history store
+job.execinfo.server.enabled=false
+job.history.store.enabled=false
+
+
diff --git a/conf/standalone/log4j-cluster.properties b/conf/cluster-master/log4j.properties
similarity index 95%
rename from conf/standalone/log4j-cluster.properties
rename to conf/cluster-master/log4j.properties
index 6effc68..8bda1cb 100755
--- a/conf/standalone/log4j-cluster.properties
+++ b/conf/cluster-master/log4j.properties
@@ -19,6 +19,7 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n
# Suppressed loggers
+log4j.logger.org.apache.helix.controller.pipeline.Pipeline=ERROR
log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR
log4j.logger.org.apache.helix.controller.stages=ERROR
log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR
diff --git a/conf/standalone/worker/quartz.properties b/conf/cluster-master/quartz.properties
similarity index 100%
rename from conf/standalone/worker/quartz.properties
rename to conf/cluster-master/quartz.properties
diff --git a/conf/standalone/master/application.conf b/conf/cluster-worker/application.conf
similarity index 78%
rename from conf/standalone/master/application.conf
rename to conf/cluster-worker/application.conf
index e9b8323..52414b0 100644
--- a/conf/standalone/master/application.conf
+++ b/conf/cluster-worker/application.conf
@@ -15,6 +15,16 @@
# limitations under the License.
#
+# Cluster configuration properties
+gobblin.cluster.app.name=GobblinStandaloneCluster
+gobblin.cluster.email.notification.on.shutdown=false
+gobblin.cluster.helix.instance.max.retries=2
+gobblin.cluster.work.dir=/tmp/gobblin-cluster
+
+# Helix/Zookeeper configuration properties
+gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
+gobblin.cluster.zk.connection.string="localhost:2181"
+
# Sample configuration properties for the Gobblin Standalone cluster
gobblin.cluster.workDir=${gobblin.cluster.work.dir}/GobblinStandaloneCluster
@@ -34,6 +44,7 @@ gobblin.cluster.job.conf.path=${gobblin.cluster.work.dir}/jobs
gobblin.cluster.jobconf.fullyQualifiedPath=${gobblin.cluster.work.dir}/jobs
# File system URIs
+fs.uri="file:///"
writer.fs.uri=${fs.uri}
state.store.fs.uri=${fs.uri}
@@ -69,3 +80,15 @@ task.status.reportintervalinms=1000
# Enable metrics / events
metrics.enabled=true
+failure.log.dir=${gobblin.cluster.work.dir}/failure-logs
+
+# UI
+admin.server.enabled=false
+# admin.server.port=9000
+
+rest.server.host=localhost
+rest.server.port=9090
+
+# job history store ( WARN [GobblinYarnAppLauncher] NOT starting the admin UI because the job execution info server is NOT enabled )
+job.execinfo.server.enabled=false
+job.history.store.enabled=false
\ No newline at end of file
diff --git a/conf/standalone/worker/log4j-cluster.properties b/conf/cluster-worker/log4j.properties
similarity index 100%
rename from conf/standalone/worker/log4j-cluster.properties
rename to conf/cluster-worker/log4j.properties
diff --git a/conf/standalone/master/quartz.properties b/conf/cluster-worker/quartz.properties
similarity index 100%
rename from conf/standalone/master/quartz.properties
rename to conf/cluster-worker/quartz.properties
diff --git a/conf/log4j-compaction.xml b/conf/compaction/log4j-compaction.xml
similarity index 100%
rename from conf/log4j-compaction.xml
rename to conf/compaction/log4j-compaction.xml
diff --git a/conf/service/application.conf b/conf/gobblin-as-service/application.conf
similarity index 91%
rename from conf/service/application.conf
rename to conf/gobblin-as-service/application.conf
index 810c6fb..7b244a1 100644
--- a/conf/service/application.conf
+++ b/conf/gobblin-as-service/application.conf
@@ -15,10 +15,13 @@
# limitations under the License.
#
-# Sample configuration properties for the Gobblin Service
+# Gobblin-As-Service configuration properties
+gobblin.service.work.dir=/tools/gobblin-dist/gobblin-as-a-service-work-dir
+fs.uri="file:///"
# Topology Catalog and Store
topologySpec.store.dir=${gobblin.service.work.dir}/topologySpecStore
+specStore.fs.dir=${gobblin.service.work.dir}/spec-store
# TopologySpec Factory
topologySpecFactory.topologyNames=localGobblinCluster
diff --git a/conf/service/log4j-service.properties b/conf/gobblin-as-service/log4j.properties
similarity index 100%
rename from conf/service/log4j-service.properties
rename to conf/gobblin-as-service/log4j.properties
diff --git a/conf/service/quartz.properties b/conf/gobblin-as-service/quartz.properties
similarity index 100%
rename from conf/service/quartz.properties
rename to conf/gobblin-as-service/quartz.properties
diff --git a/conf/gobblin-standalone-v2.properties b/conf/gobblin-standalone-v2.properties
deleted file mode 100644
index 18a8044..0000000
--- a/conf/gobblin-standalone-v2.properties
+++ /dev/null
@@ -1,65 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-###############################################################################
-###################### Gobblin standalone configurations ######################
-###############################################################################
-
-# Thread pool settings for the task executor
-taskexecutor.threadpool.size=2
-taskretry.threadpool.coresize=1
-taskretry.threadpool.maxsize=2
-
-# File system URIs
-fs.uri=file:///
-writer.fs.uri=${fs.uri}
-state.store.fs.uri=${fs.uri}
-
-# Writer related configuration properties
-writer.staging.dir=${env:GOBBLIN_WORK_DIR}/task-staging
-writer.output.dir=${env:GOBBLIN_WORK_DIR}/task-output
-
-# Data publisher related configuration properties
-data.publisher.replace.final.dir=false
-
-# Directory where job configuration files are stored
-jobconf.dir=${env:GOBBLIN_JOB_CONFIG_DIR}
-jobconf.fullyQualifiedPath=file://${env:GOBBLIN_JOB_CONFIG_DIR}
-
-# Directory where job/task state files are stored
-state.store.dir=${env:GOBBLIN_WORK_DIR}/state-store
-
-# Directory where commit sequences are stored
-gobblin.runtime.commit.sequence.store.dir=${env:GOBBLIN_WORK_DIR}/commit-sequence-store
-
-# Directory where error files from the quality checkers are stored
-qualitychecker.row.err.file=${env:GOBBLIN_WORK_DIR}/err
-
-# Directory where job locks are stored
-job.lock.dir=${env:GOBBLIN_WORK_DIR}/locks
-
-# Directory where metrics log files are stored
-metrics.log.dir=${env:GOBBLIN_WORK_DIR}/metrics
-
-# Interval of task state reporting in milliseconds
-task.status.reportintervalinms=5000
-
-# The time gap for Job Detector to detect modification/deletion/creation of jobconfig.
-# Unit in milliseconds, configurable.
-jobconf.monitor.interval=30000
-
-task.maxretries=0
diff --git a/conf/gobblin-standalone.properties b/conf/gobblin-standalone.properties
deleted file mode 100644
index 8b26fd8..0000000
--- a/conf/gobblin-standalone.properties
+++ /dev/null
@@ -1,68 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-###############################################################################
-###################### Gobblin standalone configurations ######################
-###############################################################################
-
-# Thread pool settings for the task executor
-taskexecutor.threadpool.size=2
-taskretry.threadpool.coresize=1
-taskretry.threadpool.maxsize=2
-
-# File system URIs
-fs.uri=file:///
-writer.fs.uri=${fs.uri}
-state.store.fs.uri=${fs.uri}
-
-# Writer related configuration properties
-writer.destination.type=HDFS
-writer.output.format=AVRO
-writer.staging.dir=${env:GOBBLIN_WORK_DIR}/task-staging
-writer.output.dir=${env:GOBBLIN_WORK_DIR}/task-output
-
-# Data publisher related configuration properties
-data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
-data.publisher.final.dir=${env:GOBBLIN_WORK_DIR}/job-output
-data.publisher.replace.final.dir=false
-
-# Directory where job configuration files are stored
-jobconf.dir=${env:GOBBLIN_JOB_CONFIG_DIR}
-jobconf.fullyQualifiedPath=file://${env:GOBBLIN_JOB_CONFIG_DIR}
-
-# Directory where job/task state files are stored
-state.store.dir=${env:GOBBLIN_WORK_DIR}/state-store
-
-# Directory where commit sequences are stored
-gobblin.runtime.commit.sequence.store.dir=${env:GOBBLIN_WORK_DIR}/commit-sequence-store
-
-# Directory where error files from the quality checkers are stored
-qualitychecker.row.err.file=${env:GOBBLIN_WORK_DIR}/err
-
-# Directory where job locks are stored
-job.lock.dir=${env:GOBBLIN_WORK_DIR}/locks
-
-# Directory where metrics log files are stored
-metrics.log.dir=${env:GOBBLIN_WORK_DIR}/metrics
-
-# Interval of task state reporting in milliseconds
-task.status.reportintervalinms=5000
-
-# The time gap for Job Detector to detect modification/deletion/creation of jobconfig.
-# Unit in milliseconds, configurable.
-jobconf.monitor.interval=30000
-
diff --git a/conf/log4j-mapreduce.xml b/conf/log4j-mapreduce.xml
deleted file mode 100644
index 826493a..0000000
--- a/conf/log4j-mapreduce.xml
+++ /dev/null
@@ -1,60 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration>
-
- <appender name="FileRoll" class="org.apache.log4j.rolling.RollingFileAppender">
- <param name="file" value="${gobblin.logs.dir}/gobblin-current.log" />
- <param name="append" value="true" />
- <param name="encoding" value="UTF-8" />
-
- <rollingPolicy class="org.apache.log4j.rolling.TimeBasedRollingPolicy">
- <param name="FileNamePattern" value="${gobblin.logs.dir}/archive/gobblin.%d{yyyy-MM-dd}.log"/>
- </rollingPolicy>
-
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} %L - %m%n"/>
- </layout>
- </appender>
-
- <appender name="Console" class="org.apache.log4j.ConsoleAppender">
- <param name="encoding" value="UTF-8" />
-
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} %L - %m%n"/>
- </layout>
- </appender>
-
- <logger name="org.apache.gobblin.runtime" additivity="false">
- <level value="INFO"/>
- <appender-ref ref="FileRoll" />
- </logger>
-
- <logger name="org.apache.gobblin.runtime.mapreduce.CliMRJobLauncher" additivity="false">
- <level value="ERROR"/>
- <appender-ref ref="Console" />
- </logger>
-
- <root>
- <priority value ="INFO" />
- <appender-ref ref="FileRoll" />
- </root>
-
-</log4j:configuration>
diff --git a/conf/log4j-standalone.xml b/conf/log4j-standalone.xml
deleted file mode 100644
index 63c4fa5..0000000
--- a/conf/log4j-standalone.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration>
-
- <appender name="FileRoll" class="org.apache.log4j.rolling.RollingFileAppender">
- <param name="file" value="${gobblin.logs.dir}/gobblin-current.log" />
- <param name="append" value="true" />
- <param name="encoding" value="UTF-8" />
-
- <rollingPolicy class="org.apache.log4j.rolling.TimeBasedRollingPolicy">
- <param name="FileNamePattern" value="${gobblin.logs.dir}/archive/gobblin.%d{yyyy-MM-dd}.log"/>
- </rollingPolicy>
-
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} %L - %m%n"/>
- </layout>
- </appender>
-
- <logger name="org.apache.commons.httpclient">
- <level value="DEBUG"/>
- </logger>
-
- <logger name="httpclient.wire">
- <level value="ERROR"/>
- </logger>
-
- <root>
- <priority value ="INFO" />
- <appender-ref ref="FileRoll" />
- </root>
-
-</log4j:configuration>
diff --git a/conf/log4j.properties b/conf/log4j.properties
deleted file mode 100644
index 9743fc2..0000000
--- a/conf/log4j.properties
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Root logger option
-log4j.rootLogger=INFO, stdout
-
-# Direct log messages to stdout
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.Target=System.out
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
-log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1} - %m%n
diff --git a/conf/log4j.xml b/conf/log4j.xml
deleted file mode 100644
index 873f5a7..0000000
--- a/conf/log4j.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
-
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
- <appender name="console" class="org.apache.log4j.ConsoleAppender">
- <param name="Target" value="System.out" />
- <layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern"
- value="%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} %L - %m%n" />
- </layout>
- </appender>
-
- <root>
- <level value="info" />
- <appender-ref ref="console" />
- </root>
-
-</log4j:configuration>
\ No newline at end of file
diff --git a/conf/gobblin-mapreduce.properties b/conf/mapreduce/application.conf
similarity index 78%
rename from conf/gobblin-mapreduce.properties
rename to conf/mapreduce/application.conf
index 4c704c8..9375512 100644
--- a/conf/gobblin-mapreduce.properties
+++ b/conf/mapreduce/application.conf
@@ -19,6 +19,8 @@
###################### Gobblin MapReduce configurations #######################
###############################################################################
+gobblin.cluster.work.dir=/tmp/gobblin-cluster
+
# Thread pool settings for the task executor
taskexecutor.threadpool.size=2
taskretry.threadpool.coresize=1
@@ -32,29 +34,29 @@ state.store.fs.uri=${fs.uri}
# Writer related configuration properties
writer.destination.type=HDFS
writer.output.format=AVRO
-writer.staging.dir=${env:GOBBLIN_WORK_DIR}/task-staging
-writer.output.dir=${env:GOBBLIN_WORK_DIR}/task-output
+writer.staging.dir=${gobblin.cluster.work.dir}/task-staging
+writer.output.dir=${gobblin.cluster.work.dir}/task-output
# Data publisher related configuration properties
data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
-data.publisher.final.dir=${env:GOBBLIN_WORK_DIR}/job-output
+data.publisher.final.dir=${gobblin.cluster.work.dir}/job-output
data.publisher.replace.final.dir=false
# Directory where job/task state files are stored
-state.store.dir=${env:GOBBLIN_WORK_DIR}/state-store
+state.store.dir=${gobblin.cluster.work.dir}/state-store
# Directory where error files from the quality checkers are stored
-qualitychecker.row.err.file=${env:GOBBLIN_WORK_DIR}/err
+qualitychecker.row.err.file=${gobblin.cluster.work.dir}/err
# Directory where job locks are stored
-job.lock.dir=${env:GOBBLIN_WORK_DIR}/locks
+job.lock.dir=${gobblin.cluster.work.dir}/locks
# Directory where metrics log files are stored
-metrics.log.dir=${env:GOBBLIN_WORK_DIR}/metrics
+metrics.log.dir=${gobblin.cluster.work.dir}/metrics
# Interval of task state reporting in milliseconds
task.status.reportintervalinms=5000
# MapReduce properties
-mr.job.root.dir=${env:GOBBLIN_WORK_DIR}/working
+mr.job.root.dir=${gobblin.cluster.work.dir}/working
diff --git a/conf/standalone/master/log4j-cluster.properties b/conf/mapreduce/log4j.properties
similarity index 82%
copy from conf/standalone/master/log4j-cluster.properties
copy to conf/mapreduce/log4j.properties
index 6effc68..97a3ab8 100755
--- a/conf/standalone/master/log4j-cluster.properties
+++ b/conf/mapreduce/log4j.properties
@@ -19,10 +19,13 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n
# Suppressed loggers
+log4j.logger.org.apache.gobblin.runtime.mapreduce.CliMRJobLauncher=ERROR
+log4j.logger.org.apache.helix.monitoring=ERROR
+log4j.logger.org.apache.helix.controller.pipeline.Pipeline=ERROR
log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR
log4j.logger.org.apache.helix.controller.stages=ERROR
log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR
log4j.logger.org.apache.helix.manager.zk=ERROR
-log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR
+#log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR
log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR
diff --git a/conf/service/reference.conf b/conf/service/reference.conf
deleted file mode 100644
index 7744e3e..0000000
--- a/conf/service/reference.conf
+++ /dev/null
@@ -1,5 +0,0 @@
-# Sample configuration properties with default values
-
-# Service configuration properties
-gobblin.service.work.dir=/tmp/gobblin-service
-fs.uri="file:///"
diff --git a/conf/standalone/application.conf b/conf/standalone/application.conf
index e9b8323..3a856e3 100644
--- a/conf/standalone/application.conf
+++ b/conf/standalone/application.conf
@@ -15,6 +15,19 @@
# limitations under the License.
#
+# Cluster configuration properties
+gobblin.cluster.app.name=GobblinStandaloneCluster
+gobblin.cluster.email.notification.on.shutdown=false
+gobblin.cluster.helix.instance.max.retries=2
+gobblin.cluster.work.dir=/tmp/gobblin-cluster
+
+# Helix/Zookeeper configuration properties
+gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
+gobblin.cluster.zk.connection.string="localhost:2181"
+
+# job config monitor interval
+jobconf.monitor.interval=30000
+
# Sample configuration properties for the Gobblin Standalone cluster
gobblin.cluster.workDir=${gobblin.cluster.work.dir}/GobblinStandaloneCluster
@@ -30,10 +43,11 @@ gobblin.cluster.helix.cluster.name=GobblinStandaloneClusterCli
# used by the JobConfigurationManager
gobblin.cluster.job.conf.path=${gobblin.cluster.work.dir}/jobs
-
gobblin.cluster.jobconf.fullyQualifiedPath=${gobblin.cluster.work.dir}/jobs
+gobblin.cluster.job.catalog=org.apache.gobblin.runtime.job_catalog.FSJobCatalog
# File system URIs
+fs.uri="file:///"
writer.fs.uri=${fs.uri}
state.store.fs.uri=${fs.uri}
@@ -69,3 +83,14 @@ task.status.reportintervalinms=1000
# Enable metrics / events
metrics.enabled=true
+# UI
+#admin.server.enabled=false
+admin.server.enabled=true
+admin.server.port=9000
+
+rest.server.host=localhost
+rest.server.port=9090
+
+# job history store ( WARN [GobblinYarnAppLauncher] NOT starting the admin UI because the job execution info server is NOT enabled )
+job.execinfo.server.enabled=false
+job.history.store.enabled=false
diff --git a/conf/standalone/master/log4j-cluster.properties b/conf/standalone/log4j.properties
similarity index 80%
rename from conf/standalone/master/log4j-cluster.properties
rename to conf/standalone/log4j.properties
index 6effc68..a6ff7d6 100755
--- a/conf/standalone/master/log4j-cluster.properties
+++ b/conf/standalone/log4j.properties
@@ -19,10 +19,14 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss z} %-5p [%t] %C %X{tableName} - %m%n
# Suppressed loggers
+log4j.logger.org.apache.commons.httpclient=DEBUG
+log4j.logger.org.apache.commons.httpclient.wire=ERROR
+log4j.logger.org.apache.helix.monitoring=ERROR
+log4j.logger.org.apache.helix.controller.pipeline.Pipeline=ERROR
log4j.logger.org.apache.helix.controller.GenericHelixController=ERROR
log4j.logger.org.apache.helix.controller.stages=ERROR
log4j.logger.org.apache.helix.controller.strategy.AutoRebalanceStrategy=ERROR
log4j.logger.org.apache.helix.manager.zk=ERROR
-log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR
+#log4j.logger.org.apache.helix.monitoring.mbeans.ClusterStatusMonitor=ERROR
log4j.logger.org.apache.helix.store.zk.AutoFallbackPropertyStore=ERROR
diff --git a/conf/standalone/master/reference.conf b/conf/standalone/master/reference.conf
deleted file mode 100644
index 936b8a1..0000000
--- a/conf/standalone/master/reference.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-# Sample configuration properties with default values
-
-# Cluster configuration properties
-gobblin.cluster.app.name=GobblinStandaloneCluster
-gobblin.cluster.email.notification.on.shutdown=false
-gobblin.cluster.helix.instance.max.retries=2
-gobblin.cluster.work.dir=/tmp/gobblin-cluster
-
-# Helix/Zookeeper configuration properties
-gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
-gobblin.cluster.zk.connection.string="localhost:2181"
-
-fs.uri="file:///"
-
-job.execinfo.server.enabled=false
diff --git a/conf/standalone/reference.conf b/conf/standalone/reference.conf
deleted file mode 100644
index 936b8a1..0000000
--- a/conf/standalone/reference.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-# Sample configuration properties with default values
-
-# Cluster configuration properties
-gobblin.cluster.app.name=GobblinStandaloneCluster
-gobblin.cluster.email.notification.on.shutdown=false
-gobblin.cluster.helix.instance.max.retries=2
-gobblin.cluster.work.dir=/tmp/gobblin-cluster
-
-# Helix/Zookeeper configuration properties
-gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
-gobblin.cluster.zk.connection.string="localhost:2181"
-
-fs.uri="file:///"
-
-job.execinfo.server.enabled=false
diff --git a/conf/standalone/worker/reference.conf b/conf/standalone/worker/reference.conf
deleted file mode 100644
index 936b8a1..0000000
--- a/conf/standalone/worker/reference.conf
+++ /dev/null
@@ -1,15 +0,0 @@
-# Sample configuration properties with default values
-
-# Cluster configuration properties
-gobblin.cluster.app.name=GobblinStandaloneCluster
-gobblin.cluster.email.notification.on.shutdown=false
-gobblin.cluster.helix.instance.max.retries=2
-gobblin.cluster.work.dir=/tmp/gobblin-cluster
-
-# Helix/Zookeeper configuration properties
-gobblin.cluster.helix.cluster.name=GobblinStandaloneCluster
-gobblin.cluster.zk.connection.string="localhost:2181"
-
-fs.uri="file:///"
-
-job.execinfo.server.enabled=false
diff --git a/conf/yarn/application.conf b/conf/yarn/application.conf
index ebf6187..b31a731 100644
--- a/conf/yarn/application.conf
+++ b/conf/yarn/application.conf
@@ -17,22 +17,42 @@
# Sample configuration properties for the Gobblin Yarn application launcher
+# Sample configuration properties with default values
+
+
+
# Yarn/Helix configuration properties
+gobblin.yarn.app.queue=default
gobblin.yarn.app.name=GobblinYarn
-gobblin.yarn.app.master.memory.mbs=256
+gobblin.yarn.app.master.memory.mbs=512
+gobblin.yarn.app.master.cores=1
+gobblin.yarn.app.report.interval.minutes=5
+gobblin.yarn.max.get.app.report.failures=4
gobblin.yarn.initial.containers=2
+gobblin.yarn.container.cores=1
gobblin.yarn.container.memory.mbs=512
-gobblin.yarn.conf.dir=<directory where Gobblin on Yarn related configuration files are located>
-gobblin.yarn.lib.jars.dir=<directory where Gobblin on Yarn lib jars are located>
-gobblin.yarn.app.master.files.local=${gobblin.yarn.conf.dir}"/log4j-yarn.properties,"${gobblin.yarn.conf.dir}"/application.conf,"${gobblin.yarn.conf.dir}"/reference.conf"
+gobblin.yarn.container.affinity.enabled=true
+gobblin.yarn.keytab.login.interval.minutes=1440
+gobblin.yarn.token.renew.interval.minutes=720
+gobblin.yarn.helix.instance.max.retries=2
+
+gobblin.yarn.work.dir=/tmp/gobblin
+gobblin.yarn.conf.dir=${GOBBLIN_HOME}/conf/yarn/
+gobblin.yarn.lib.jars.dir=${GOBBLIN_HOME}/lib/
+gobblin.yarn.app.master.files.local=${gobblin.yarn.conf.dir}"/log4j.properties,"${gobblin.yarn.conf.dir}"/application.conf,"${gobblin.yarn.conf.dir}"/reference.conf"
gobblin.yarn.container.files.local=${gobblin.yarn.app.master.files.local}
-gobblin.yarn.logs.sink.root.dir=<root sink directory for aggregated application/container logs stored on the launcher side>
+gobblin.yarn.logs.sink.root.dir=/tools/gobblin-dist/yarn_app_logs
+gobblin.yarn.email.notification.on.shutdown=false
+
+gobblin.cluster.zk.connection.string="localhost:2181"
+
# Cluster configuration properties
gobblin.cluster.helix.cluster.name=GobblinYarn
-gobblin.cluster.job.conf.path=<path where Gobblin job configuration file are located>
+gobblin.cluster.job.conf.path=/tools/gobblin-dist/job_conf
# File system URIs
+fs.uri="hdfs://localhost:8020"
writer.fs.uri=${fs.uri}
state.store.fs.uri=${fs.uri}
@@ -68,3 +88,10 @@ task.status.reportintervalinms=1000
# Enable metrics / events
metrics.enabled=true
+# UI
+admin.server.enabled=false
+# admin.server.port=9000
+
+# job history store ( WARN [GobblinYarnAppLauncher] NOT starting the admin UI because the job execution info server is NOT enabled )
+job.execinfo.server.enabled=false
+job.history.store.enabled=false
diff --git a/conf/aws/log4j-aws.properties b/conf/yarn/log4j.properties
similarity index 100%
rename from conf/aws/log4j-aws.properties
rename to conf/yarn/log4j.properties
diff --git a/conf/yarn/quartz.properties b/conf/yarn/quartz.properties
deleted file mode 120000
index da8e22c..0000000
--- a/conf/yarn/quartz.properties
+++ /dev/null
@@ -1 +0,0 @@
-../quartz.properties
\ No newline at end of file
diff --git a/conf/yarn/quartz.properties b/conf/yarn/quartz.properties
new file mode 100755
index 0000000..0ea5f30
--- /dev/null
+++ b/conf/yarn/quartz.properties
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+org.quartz.scheduler.instanceName = LocalJobScheduler
+org.quartz.threadPool.threadCount = 3
+org.quartz.jobStore.class = org.quartz.simpl.RAMJobStore
diff --git a/conf/yarn/reference.conf b/conf/yarn/reference.conf
deleted file mode 100644
index e26885c..0000000
--- a/conf/yarn/reference.conf
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Sample configuration properties with default values
-
-# Yarn/Helix configuration properties
-gobblin.yarn.app.queue=default
-gobblin.yarn.app.name=GobblinYarn
-gobblin.yarn.app.master.memory.mbs=512
-gobblin.yarn.app.master.cores=1
-gobblin.yarn.app.report.interval.minutes=5
-gobblin.yarn.max.get.app.report.failures=4
-gobblin.yarn.email.notification.on.shutdown=false
-gobblin.yarn.initial.containers=1
-gobblin.yarn.container.memory.mbs=512
-gobblin.yarn.container.cores=1
-gobblin.yarn.container.affinity.enabled=true
-gobblin.yarn.helix.instance.max.retries=2
-gobblin.yarn.keytab.login.interval.minutes=1440
-gobblin.yarn.token.renew.interval.minutes=720
-gobblin.yarn.work.dir=/gobblin
-
-# Cluster configuration properties
-gobblin.cluster.helix.cluster.name=GobblinYarn
-gobblin.cluster.zk.connection.string="localhost:2181"
-
-fs.uri="hdfs://localhost:9000"
-
-job.execinfo.server.enabled=false
diff --git a/gobblin-admin/src/main/java/org/apache/gobblin/cli/Cli.java b/gobblin-admin/src/main/java/org/apache/gobblin/cli/Cli.java
deleted file mode 100644
index 61ca72d..0000000
--- a/gobblin-admin/src/main/java/org/apache/gobblin/cli/Cli.java
+++ /dev/null
@@ -1,196 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.gobblin.cli;
-
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Map;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-
-import com.google.common.collect.ImmutableMap;
-
-/**
- * A command line interface for interacting with Gobblin.
- * From this tool, you should be able to:
- * * Check the status of Gobblin jobs
- * * View ...
- *
- * @author ahollenbach@nerdwallet.com
- */
-public class Cli {
- private static final Map<String, Command> commandList =
- ImmutableMap.of(
- "jobs", (Command)new JobCommand()
- );
-
- static class GlobalOptions {
- private final String adminServerHost;
- private final int adminServerPort;
-
- public GlobalOptions(String adminServerHost, int adminServerPort) {
- this.adminServerHost = adminServerHost;
- this.adminServerPort = adminServerPort;
- }
-
- public String getAdminServerHost() {
- return adminServerHost;
- }
-
- public int getAdminServerPort() {
- return adminServerPort;
- }
- }
-
- /**
- * Get the list of valid command names
- * @return List of command names
- */
- public static Collection<String> getCommandNames() {
- return commandList.keySet();
- }
-
- // Option long codes
- private static final String HOST_OPT = "host";
- private static final String PORT_OPT = "port";
-
- private static final String DEFAULT_REST_SERVER_HOST = "localhost";
- private static final int DEFAULT_REST_SERVER_PORT = 8080;
-
-
- private String[] args;
- private Options options;
-
- public static void main(String[] args) {
- Cli cli = new Cli(args);
- cli.parseAndExecuteCommand();
- }
-
- /**
- * Create a new Cli object.
- * @param args Command line arguments
- */
- public Cli(String[] args) {
- this.args = args;
-
- this.options = new Options();
-
- this.options.addOption("H", HOST_OPT, true, "Specify host (default:" + DEFAULT_REST_SERVER_HOST + ")");
- this.options.addOption("P", PORT_OPT, true, "Specify port (default:" + DEFAULT_REST_SERVER_PORT + ")");
- }
-
- /**
- * Parse and execute the appropriate command based on the args.
- * The general flow looks like this:
- *
- * 1. Parse a set of global options (eg host/port for the admin server)
- * 2. Parse out the command name
- * 3. Pass the global options and any left over parameters to a command handler
- */
- public void parseAndExecuteCommand() {
- CommandLineParser parser = new DefaultParser();
- try {
- CommandLine parsedOpts = parser.parse(this.options, this.args, true);
- GlobalOptions globalOptions = createGlobalOptions(parsedOpts);
-
- // Fetch the command and fail if there is ambiguity
- String[] remainingArgs = parsedOpts.getArgs();
- if (remainingArgs.length == 0) {
- printHelpAndExit("Command not specified!");
- }
-
- String commandName = remainingArgs[0].toLowerCase();
- remainingArgs = remainingArgs.length > 1 ?
- Arrays.copyOfRange(remainingArgs, 1, remainingArgs.length) :
- new String[]{};
-
- Command command = commandList.get(commandName);
- if (command == null) {
- System.out.println("Command " + commandName + " not known.");
- printHelpAndExit();
- } else {
- command.execute(globalOptions, remainingArgs);
- }
- } catch (ParseException e) {
- printHelpAndExit("Ran into an error parsing args.");
- }
- }
-
- /**
- * Build the GlobalOptions information from the raw parsed options
- * @param parsedOpts Options parsed from the cmd line
- * @return
- */
- private GlobalOptions createGlobalOptions(CommandLine parsedOpts) {
- String host = parsedOpts.hasOption(HOST_OPT) ?
- parsedOpts.getOptionValue(HOST_OPT) : DEFAULT_REST_SERVER_HOST;
- int port = DEFAULT_REST_SERVER_PORT;
- try {
- if (parsedOpts.hasOption(PORT_OPT)) {
- port = Integer.parseInt(parsedOpts.getOptionValue(PORT_OPT));
- }
- } catch (NumberFormatException e) {
- printHelpAndExit("The port must be a valid integer.");
- }
-
- return new GlobalOptions(host, port);
- }
-
- /**
- * Print help and exit with a success code (0).
- */
- private void printHelpAndExit() {
- System.out.println("Common usages:");
- System.out.println(" gobblin-admin.sh jobs --list");
- System.out.println(" gobblin-admin.sh jobs --list --name JobName");
- System.out.println(" gobblin-admin.sh jobs --details --id job_id");
- System.out.println(" gobblin-admin.sh jobs --properties --<id|name> <job_id|JobName>");
- System.out.println();
-
- printHelpAndExit(0);
- }
-
- /**
- * Prints an error message, then prints the help and exits with an error code.
- */
- private void printHelpAndExit(String errorMessage) {
- System.err.println(errorMessage);
- printHelpAndExit(1);
- }
-
- /**
- * Print help and exit with the specified code.
- * @param exitCode The code to exit with
- */
- private void printHelpAndExit(int exitCode) {
- HelpFormatter hf = new HelpFormatter();
-
- hf.printHelp("gobblin-admin.sh <command> [options]", this.options);
- System.out.println("Valid commands:");
- for (String command : getCommandNames()) {
- System.out.println(command);
- }
-
- System.exit(exitCode);
- }
-}
diff --git a/gobblin-admin/src/main/java/org/apache/gobblin/cli/Command.java b/gobblin-admin/src/main/java/org/apache/gobblin/cli/Command.java
deleted file mode 100644
index 6598da4..0000000
--- a/gobblin-admin/src/main/java/org/apache/gobblin/cli/Command.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.gobblin.cli;
-
-/**
- * Represents a single command for the CLI
- */
-public interface Command {
- void execute(Cli.GlobalOptions globalOptions, String[] otherArgs);
-}
diff --git a/gobblin-admin/src/main/java/org/apache/gobblin/cli/JobCommand.java b/gobblin-admin/src/main/java/org/apache/gobblin/cli/JobCommand.java
index bb37012..73baae1 100644
--- a/gobblin-admin/src/main/java/org/apache/gobblin/cli/JobCommand.java
+++ b/gobblin-admin/src/main/java/org/apache/gobblin/cli/JobCommand.java
@@ -19,6 +19,7 @@ package org.apache.gobblin.cli;
import java.util.List;
import java.util.Map;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.DefaultParser;
import org.apache.commons.cli.HelpFormatter;
@@ -31,13 +32,17 @@ import com.google.common.base.Optional;
import com.google.common.collect.ImmutableMap;
import com.linkedin.r2.RemoteInvocationException;
+import org.apache.gobblin.annotation.Alias;
import org.apache.gobblin.rest.JobExecutionInfo;
import org.apache.gobblin.rest.QueryListType;
+import org.apache.gobblin.runtime.cli.CliApplication;
/**
* Logic to print out job state
*/
-public class JobCommand implements Command {
+@Slf4j
+@Alias(value = "jobs", description = "Command line job info and operations")
+public class JobCommand implements CliApplication {
private Options options;
private static class CommandException extends Exception {
@@ -53,6 +58,10 @@ public class JobCommand implements Command {
throws CommandException;
}
+ private static final String ADMIN_SERVER = "host";
+ private static final String DEFAULT_ADMIN_SERVER = "localhost";
+ private static final int DEFAULT_ADMIN_PORT = 8080;
+ private static final String ADMIN_PORT = "port";
private static final String HELP_OPT = "help";
private static final String DETAILS_OPT = "details";
private static final String LIST_OPT = "list";
@@ -79,27 +88,38 @@ public class JobCommand implements Command {
}
}
- printHelpAndExit("Unknown subcommand");
+ printHelpAndExit("Unknown subcommand", false);
throw new IllegalStateException("unreached...");
}
@Override
- public void execute(Cli.GlobalOptions globalOptions, String[] otherArgs) {
+ public void run(String[] args) throws Exception {
this.options = createCommandLineOptions();
DefaultParser parser = new DefaultParser();
AdminClient adminClient = null;
try {
- CommandLine parsedOpts = parser.parse(options, otherArgs);
+ CommandLine parsedOpts = parser.parse(options, args);
int resultLimit = parseResultsLimit(parsedOpts);
- adminClient = new AdminClient(globalOptions.getAdminServerHost(), globalOptions.getAdminServerPort());
+ String host = parsedOpts.hasOption(ADMIN_SERVER) ?
+ parsedOpts.getOptionValue(ADMIN_SERVER) : DEFAULT_ADMIN_SERVER;
+ int port = DEFAULT_ADMIN_PORT;
+ try {
+ if (parsedOpts.hasOption(ADMIN_PORT)) {
+ port = Integer.parseInt(parsedOpts.getOptionValue(ADMIN_PORT));
+ }
+ } catch (NumberFormatException e) {
+ printHelpAndExit("The port must be a valid integer.", false);
+ }
+
+ adminClient = new AdminClient(host, port);
try {
getAction(parsedOpts).execute(parsedOpts, adminClient, resultLimit);
} catch (CommandException e) {
- printHelpAndExit(e.getMessage());
+ printHelpAndExit(e.getMessage(), false);
}
} catch (ParseException e) {
- printHelpAndExit("Failed to parse jobs arguments: " + e.getMessage());
+ printHelpAndExit("Failed to parse jobs arguments: " + e.getMessage(), true);
} finally {
if (adminClient != null) adminClient.close();
}
@@ -183,6 +203,8 @@ public class JobCommand implements Command {
options.addOption("n", true, "Limit the number of results returned. (default:" + DEFAULT_RESULTS_LIMIT + ")");
options.addOption("r", RECENT_OPT, false, "List the most recent jobs (instead of a list of unique jobs)");
+ options.addOption("H", ADMIN_SERVER, true, "hostname of admin server");
+ options.addOption("P", ADMIN_PORT, true, "port of admin server");
return options;
}
@@ -192,7 +214,7 @@ public class JobCommand implements Command {
try {
return Integer.parseInt(parsedOpts.getOptionValue("n"));
} catch (NumberFormatException e) {
- printHelpAndExit("Could not parse integer value for option n.");
+ printHelpAndExit("Could not parse integer value for option n.", false);
return 0;
}
} else {
@@ -203,13 +225,12 @@ public class JobCommand implements Command {
/**
* Print help and exit with the specified code.
*/
- private void printHelpAndExit(String errorMsg) {
+ private void printHelpAndExit(String errorMsg, boolean printHelp) {
System.out.println(errorMsg);
-
- HelpFormatter hf = new HelpFormatter();
-
- hf.printHelp("gobblin-admin.sh jobs [options]", this.options);
-
+ if (printHelp) {
+ HelpFormatter hf = new HelpFormatter();
+ hf.printHelp("gobblin-admin.sh jobs [options]", this.options);
+ }
System.exit(1);
}
}
diff --git a/gobblin-docs/Getting-Started.md b/gobblin-docs/Getting-Started.md
index d575997..5f5d843 100644
--- a/gobblin-docs/Getting-Started.md
+++ b/gobblin-docs/Getting-Started.md
@@ -1,3 +1,4 @@
+
Table of Contents
-----------------
@@ -54,18 +55,18 @@ This page explains how to run the job from the terminal. You may also run this j
## Steps
-* cd to the unpacked Gobblin distribution and run `bin/gobblin run` to get usage.
-* Running `bin/gobblin run listQuickApps` will list the available easy-to-configure apps. Note the line with the wikipedia example:
+* cd to the unpacked Gobblin distribution and run `bin/gobblin cli run` to get usage.
+* Running `bin/gobblin cli run listQuickApps` will list the available easy-to-configure apps. Note the line with the wikipedia example:
```bash
wikipedia - Gobblin example that downloads revisions from Wikipedia.
```
-* Running `bin/gobblin run wikipedia` will show the usage of this application. Notice the usage and one of the options listed for this job:
+* Running `bin/gobblin cli run wikipedia` will show the usage of this application. Notice the usage and one of the options listed for this job:
```bash
-usage: gobblin run wikipedia [OPTIONS] <article-title> [<article-title>...]
+usage: gobblin cli run wikipedia [OPTIONS] <article-title> [<article-title>...]
-lookback <arg> Sets the period for which articles should be
pulled in ISO time format (e.g. P2D, PT1H)
```
-* Run `bin/gobblin run wikipedia -lookback P10D LinkedIn Wikipedia:Sandbox`. This will print a lot of logs, but somewhere in there you will see a few json entries with the revisions for those articles. For example:
+* Run `bin/gobblin cli run wikipedia -lookback P10D LinkedIn Wikipedia:Sandbox`. This will print a lot of logs, but somewhere in there you will see a few json entries with the revisions for those articles. For example:
```bash
{"revid":746260034,"parentid":745444076,"user":"2605:8D80:580:5824:B108:82BD:693D:CFA1","anon":"","userid":0,"timestamp":"2016-10-26T08:12:09Z","size":69527,"pageid":970755,"title":"LinkedIn"}
```
@@ -74,7 +75,7 @@ usage: gobblin run wikipedia [OPTIONS] <article-title> [<article-title>...]
-avroOutput <arg> Write output to Avro files. Specify the
output directory as argument.
```
-Running `bin/gobblin run wikipedia -lookback P10D -avroOutput /tmp/wikiSample LinkedIn Wikipedia:Sandbox` will create a directory `/tmp/wikiSample` with two subdirectories `LinkedIn` and `Wikipedia_Sandbox` each one with one avro file.
+Running `bin/gobblin cli run wikipedia -lookback P10D -avroOutput /tmp/wikiSample LinkedIn Wikipedia:Sandbox` will create a directory `/tmp/wikiSample` with two subdirectories `LinkedIn` and `Wikipedia_Sandbox` each one with one avro file.
# Running Gobblin as a Daemon
@@ -100,15 +101,13 @@ A list of commonly used configuration properties can be found here: [Configurati
* Unpack Gobblin distribution:
-* Launch Gobblin:
+* Launch Gobblin in one of the execution mode [for more info refer: [Gobblin-CLI](/gobblin-docs/user-guide/Gobblin-CLI.md)] :
```bash
-bin/gobblin-standalone.sh start
+gobblin service standalone start
```
-This script will launch Gobblin and pass the Gobblin configuration file ([gobblin-standalone.properties](https://github.com/apache/incubator-gobblin/blob/master/conf/gobblin-standalone-v2.properties)) as an argument.
-
-The job log, which contains the progress and status of the job, will be written into `logs/gobblin-current.log` (to change where the log is written, modify the Log4j configuration file `conf/log4j-standalone.xml`). Stdout will be written into `nohup.out`.
+The job log, which contains the progress and status of the job, will be written into `logs/<execution-mode>.out` & `logs/<execution-mode>.err` (to change where the log is written, modify the Log4j configuration file `conf/log4j.properties`).
Among the job logs there should be the following information:
@@ -138,7 +137,7 @@ INFO AbstractJobLauncher - Persisting job/task states of job job_PullFromWikipe
* After the job is done, stop Gobblin by running
```bash
-bin/gobblin-standalone.sh stop
+gobblin service standalone stop
```
The job output is written in `GOBBLIN_WORK_DIR/job-output` folder as an Avro file.
diff --git a/gobblin-docs/Gobblin-Architecture.md b/gobblin-docs/Gobblin-Architecture.md
index 07abb01..4e5f7a3 100644
--- a/gobblin-docs/Gobblin-Architecture.md
+++ b/gobblin-docs/Gobblin-Architecture.md
@@ -137,4 +137,5 @@ As a fault tolerance data ingestion framework, Gobblin employs multiple level of
Job Scheduling
--------------------------------
-Like mentioned above, a Gobblin job typically runs periodically on some schedule. Gobblin can be integrated with job schedulers such as [Azkaban](http://azkaban.github.io/),[Oozie](http://oozie.apache.org/), or Crontab. Out-of-the-box, Gobblin also ships with a built-in job scheduler backed by a [Quartz](http://quartz-scheduler.org/) scheduler, which is used as the default job scheduler in the standalone deployment. An important feature of Gobblin is that it decouples the job scheduler a [...]
+Like mentioned above, a Gobblin job typically runs periodically on some schedule. Gobblin can be integrated with job schedulers such as [Azkaban](http://azkaban.github.io/),[Oozie](http://oozie.apache.org/), or Crontab. Out-of-the-box, Gobblin also ships with a built-in job scheduler backed by a [Quartz](http://quartz-scheduler.org/) scheduler, which is used as the default job scheduler in the standalone deployment and it supports cron-based triggers using the configuration property `job [...]
+
diff --git a/gobblin-docs/case-studies/Kafka-HDFS-Ingestion.md b/gobblin-docs/case-studies/Kafka-HDFS-Ingestion.md
index 352b7e3..87817f4 100644
--- a/gobblin-docs/case-studies/Kafka-HDFS-Ingestion.md
+++ b/gobblin-docs/case-studies/Kafka-HDFS-Ingestion.md
@@ -34,7 +34,7 @@ data.publisher.type=org.apache.gobblin.publisher.BaseDataPublisher
mr.job.max.mappers=1
metrics.reporting.file.enabled=true
-metrics.log.dir=${env:GOBBLIN_WORK_DIR}/metrics
+metrics.log.dir=${gobblin.cluster.work.dir}/metrics
metrics.reporting.file.suffix=txt
bootstrap.with.offset=earliest
@@ -52,7 +52,7 @@ INFO Actual high watermark for partition test:0=2, expected=2
INFO Task <task_id> completed in 31212ms with state SUCCESSFUL
```
-The output file will be in `GOBBLIN_WORK_DIR/job-output/test`, with the two messages you've just created in the Kafka broker. `GOBBLIN_WORK_DIR/metrics` will contain metrics collected from this run.
+The output file will be in `{gobblin.cluster.work.dir}/job-output/test`, with the two messages you've just created in the Kafka broker. `{gobblin.cluster.work.dir}/metrics` will contain metrics collected from this run.
## MapReduce
diff --git a/gobblin-docs/case-studies/Publishing-Data-to-S3.md b/gobblin-docs/case-studies/Publishing-Data-to-S3.md
index 9866e85..cbdb8ad 100644
--- a/gobblin-docs/case-studies/Publishing-Data-to-S3.md
+++ b/gobblin-docs/case-studies/Publishing-Data-to-S3.md
@@ -95,7 +95,7 @@ Go to the [S3 dashboard](https://console.aws.amazon.com/s3)
## Setting Up Gobblin on EC2
* Download and Build Gobblin Locally
- * On your local machine, clone the [Gobblin repository](https://github.com/apache/incubator-gobblin): `git clone git@github.com:linkedin/gobblin.git` (this assumes you have [Git](https://en.wikipedia.org/wiki/Git_(software)) installed locally)
+ * On your local machine, clone the [Gobblin repository](https://github.com/apache/incubator-gobblin): `git clone git@github.com:apache/incubator-gobblin.git` (this assumes you have [Git](https://en.wikipedia.org/wiki/Git_(software)) installed locally)
* Build Gobblin using the following commands (it is important to use Hadoop version 2.6.0 as it includes the `s3a` file system implementation):
```
cd gobblin
@@ -134,7 +134,7 @@ Assuming we want Gobblin to run in standalone mode, follow the usual steps for [
For the sake of this walkthrough, we will launch the Gobblin [wikipedia example](https://github.com/apache/incubator-gobblin/blob/master/gobblin-example/src/main/resources/wikipedia.pull). Directions on how to run this example can be found [here](../Getting-Started). The command to launch Gobblin should look similar to:
```
-sh bin/gobblin-standalone.sh start --workdir /home/ec2-user/gobblin-dist/work --logdir /home/ec2-user/gobblin-dist/logs --conf /home/ec2-user/gobblin-dist/config
+sh bin/gobblin standalone start --conf-dir /home/ec2-user/gobblin-dist/config
```
If you are running on the Amazon free tier, you will probably get an error in the `nohup.out` file saying there is insufficient memory for the JVM. To fix this add `--jvmflags "-Xms256m -Xmx512m"` to the `start` command.
diff --git a/gobblin-docs/sinks/AvroHdfsDataWriter.md b/gobblin-docs/sinks/AvroHdfsDataWriter.md
index 8ac89fb..2140ad8 100644
--- a/gobblin-docs/sinks/AvroHdfsDataWriter.md
+++ b/gobblin-docs/sinks/AvroHdfsDataWriter.md
@@ -9,7 +9,7 @@ Writes Avro records to Avro data files on Hadoop file systems.
writer.builder.class=org.apache.gobblin.writer.AvroDataWriterBuilder
writer.destination.type=HDFS
-For more info, see [`AvroHdfsDataWriter`](https://github.com/apache/incubator-gobblin/blob/master/gobblin-core/src/main/java/org/apache/gobblin/writer/AvroHdfsDataWriter.java)
+For more info, see [`AvroHdfsDataWriter`](https://github.com/apache/incubator-gobblin/search?utf8=%E2%9C%93&q=AvroHdfsDataWriter)
# Configuration
diff --git a/gobblin-docs/user-guide/Gobblin-CLI.md b/gobblin-docs/user-guide/Gobblin-CLI.md
index 4a14893..d3a434a 100644
--- a/gobblin-docs/user-guide/Gobblin-CLI.md
+++ b/gobblin-docs/user-guide/Gobblin-CLI.md
@@ -3,80 +3,127 @@ Table of Contents
[TOC]
-Gobblin CLI
+Gobblin Commands & Execution Modes
-----------
-The Gobblin distribution contains a CLI at `bin/gobblin` providing CLI access to various Gobblin applications and commands. To run:
+The Gobblin distribution comes with a script `./bin/gobblin` for all commands and services.
+Here is the usage:
+
```bash
-bin/gobblin
+Usage:
+gobblin.sh cli <cli-command> <params>
+gobblin.sh service <execution-mode> <start|stop|status>
+
+Use "gobblin <cli|service> --help" for more information. (Gobblin Version: 0.15.0)
```
-The usage is `bin/gobblin <command>` where the command specifies an application to run. Running `bin/gobblin -h` provides a list of available commands.
-The special command `bin/gobblin classpath` is trapped by the bash script and simply displays the full classpath that Gobblin uses.
+For Gobblin CLI commands, run following:
+```bash
+Usage:
+gobblin.sh cli <cli-command> <params>
+
+options:
+ cli-commands:
+ passwordManager Encrypt or decrypt strings for the password manager.
+ decrypt Decryption utilities
+ run Run a Gobblin application.
+ config Query the config library
+ jobs Command line job info and operations
+ stateMigration Command line tools for migrating state store
+ job-state-to-json To convert Job state to JSON
+ cleaner Data retention utility
+ keystore Examine JCE Keystore files
+ watermarks Inspect streaming watermarks
+ job-store-schema-manager Database job history store schema manager
+
+ --conf-dir <gobblin-conf-dir-path> Gobblon config path. default is '$GOBBLIN_HOME/conf/<exe-mode-name>'.
+ --log4j-conf <path-of-log4j-file> default is '<gobblin-conf-dir-path>/<execution-mode>/log4j.properties'.
+ --jvmopts <jvm or gc options> String containing JVM flags to include, in addition to "-Xmx1g -Xms512m".
+ --jars <csv list of extra jars> Column-separated list of extra jars to put on the CLASSPATH.
+ --enable-gc-logs enables gc logs & dumps.
+ --show-classpath prints gobblin runtime classpath.
+ --help Display this help.
+ --verbose Display full command used to start the process.
+ Gobblin Version: 0.15.0
+```
-If running from an IDE, the main method for the CLI is `gobblin.runtime.cli.GobblinCli`.
-A note on Hadoop classpath
--------------------------
+Argument details:
+* `--conf-dir`: specifies the path to directory containing gobblin system configuration files, like `application.conf` or `reference.conf`, `log4j.properties` and `quartz.properties`.
+* `--log4j-conf`: specify the path of log4j config file to override the one in config directory (default is `<conf>/<gobblin-mode>/log4j.properties`. Gobblin uses [SLF4J](http://www.slf4j.org/) and the [slf4j-log4j12](http://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12) binding for logging.
+* `--jvmopts`: to specify any JVM parameters, default is `-Xmx1g -Xms512m`.
+* `--enable-gc-logs`: adds GC options to JVM parameters: ``` -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseCompressedOops -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=$GOBBLIN_LOGS/ -Xloggc:$GOBBLIN_LOGS/gobblin-$GOBBLIN_MODE-gc.log ```
+* `--show-classpath`: It prints the full value of the classpath that gobblin uses.
+* all other arguments are sell-explanatory.
-When running `bin/gobblin`, the script automatically finds the classpath for the job. Although Hadoop jars are included in the Gobblin distribution, if `HADOOP_HOME` is set in the environment, Gobblin will instead use the classpath provided by the local Hadoop installation.
+Gobblin Commands
+-------------------
-An important side effect of this is that (depending on the application) non-fully-qualified paths (like `/my/path`) will default to local file system if `HADOOP_HOME` is not set, while they will default to HDFS if the variable is set. When referring to local paths, it is always a good idea to use the fully qualified path (e.g. `file:///my/path`).
+Gobblin provides following CLI commands:
-Gobblin ingestion applications
------------------------------
+ Available commands:
+ job-state-to-json To convert Job state to JSON
+ jobs Command line job info and operations
+ passwordManager Encrypt or decrypt strings for the password manager.
+ run Run a Gobblin application.
+ decrypt Decryption utilities
+ job-store-schema-manager Database job history store schema manager
+ stateMigration Command line tools for migrating state store
+ keystore Examine JCE Keystore files
+ config Query the config library
+ watermarks Inspect streaming watermarks
+ cleaner Data retention utility
-Gobblin ingestion applications can be accessed through the command `run`:
-```bash
-bin/gobblin run [listQuickApps] [<quick-app>] -jobName <jobName> [OPTIONS]
-```
-For usage run `bin/gobblin run`.
+Details on how to use `run` command:
+
+Gobblin ingestion applications can be accessed through the following command:
-`bin/gobblin run` uses [Embedded Gobblin](Gobblin-as-a-Library.md) and subclasses to run Gobblin ingestion jobs, giving CLI access to most functionality that could be achieved using `EmbeddedGobblin`. For example, the following command will run a Hello World job (it will print "Hello World 1 !" somewhere in the logs).
-```bash
-bin/gobblin run -jobName helloWorld -setTemplate resource:///templates/hello-world.template
-```
+ gobblin cli run [listQuickApps] [<quick-app>] -jobName <jobName> [OPTIONS]
+
+For usage run `./bin/gobblin cli run`.
+
+`gobblin cli run` uses [Embedded Gobblin](Gobblin-as-a-Library.md) and subclasses to run Gobblin ingestion jobs, giving CLI access to most functionality that could be achieved using `EmbeddedGobblin`. For example, the following command will run a Hello World job (it will print "Hello World 1 !" somewhere in the logs).
+
+ gobblin cli run -jobName helloWorld -setTemplate resource:///templates/hello-world.template
Obviously, it is daunting to have to know the path to templates and exactly which configurations to set. The alternative is to use a quick app. Running:
-```bash
-bin/gobblin run listQuickApps
-```
+
+ gobblin cli run listQuickApps
+
will provide with a list of available quick apps. To run a quick app:
-```bash
-bin/gobblin run <quick-app-name>
-```
-Quick apps may require additional arguments. For the usage of a particular app, run `bin/gobblin run <quick-app-name> -h`.
+
+ gobblin cli run <quick-app-name>
+
+Quick apps may require additional arguments. For the usage of a particular app, run `bin/gobblin cli run <quick-app-name> -h`.
For example, consider the quick app distcp:
```bash
-$ bin/gobblin run distcp -h
-usage: gobblin run distcp [OPTIONS] <source> <target>
+$ gobblin cli run distcp -h
+usage: gobblin cli run distcp [OPTIONS] <source> <target>
-delete Delete files in target that don't exist
on source.
-deleteEmptyParentDirectories If deleting files on target, also delete
newly empty parent directories.
-distributeJar <arg>
-h,--help
- -l Uses log to print out erros in the base
- CLI code.
+ -l Uses log to print out erros in the base CLI code.
-mrMode
-setConfiguration <arg>
-setJobTimeout <arg>
-setLaunchTimeout <arg>
-setShutdownTimeout <arg>
-simulate
- -update Specifies files should be updated if
- they're different in the source.
+ -update Specifies files should be updated if they're different in the source.
-useStateStore <arg>
```
This provides usage for the app distcp, as well as listing all available options. Distcp could then be run:
```bash
-bin/gobblin run distcp file:///source/path file:///target/path
+gobblin cli run distcp file:///source/path file:///target/path
```
Developing quick apps for the CLI
--------------------------------------------
-
+
It is very easy to convert a subclass of `EmbeddedGobblin` into a quick application for Gobblin CLI. All that is needed is to implement a `EmbeddedGobblinCliFactory` which knows how instantiate the `EmbeddedGobblin` from a `CommandLine` object and annotate it with the `Alias` annotation. There are two utility classes that make this very easy:
* `PublicMethodsGobblinCliFactory`: this class will automatically infer CLI options from the public methods of a subclass of `EmbeddedGobblin`. All the developer has to do is implement the method `constructEmbeddedGobblin(CommandLine)` that calls the appropriate constructor of the desired `EmbeddedGobblin` subclass with parameters extracted from the CLI. Additionally, it is a good idea to override `getUsageString()` with the appropriate usage string. For an example, see `gobblin.runtime. [...]
@@ -85,4 +132,89 @@ It is very easy to convert a subclass of `EmbeddedGobblin` into a quick applicat
Implementing new Gobblin commands
---------------------------------
-To implement a new Gobblin command to list and execute using `bin/gobblin`, implement the class `gobblin.runtime.cli.CliApplication`, and annotate it with the `Alias` annotation. The Gobblin CLI will automatically find the command, and users can invoke it by the Alias value.
+To implement a new Gobblin command to list and execute using `./bin/gobblin`, implement the class `gobblin.runtime.cli.CliApplication`, and annotate it with the `Alias` annotation. The Gobblin CLI will automatically find the command, and users can invoke it by the Alias value.
+
+
+
+Gobblin Service Execution Modes ( as Daemon )
+-------------------
+
+For more info on Gobblin service execution modes, run `bin/gobblin service --help`:
+```bash
+Usage:
+gobblin.sh service <execution-mode> <start|stop|status>
+
+Argument Options:
+ <execution-mode> standalone, cluster-master, cluster-worker, aws,
+ yarn, mapreduce, service-manager.
+
+ --conf-dir <gobblin-conf-dir-path> Gobblon config path. default is '$GOBBLIN_HOME/conf/<exe-mode-name>'.
+ --log4j-conf <path-of-log4j-file> default is '<gobblin-conf-dir-path>/<execution-mode>/log4j.properties'. --jvmopts <jvm or gc options> String containing JVM flags to include, in addition to "-Xmx1g -Xms512m".
+ --jars <csv list of extra jars> Column-separated list of extra jars to put on the CLASSPATH.
+ --enable-gc-logs enables gc logs & dumps.
+ --show-classpath prints gobblin runtime classpath.
+ --cluster-name Name of the cluster to be used by helix & other services. ( default: gobblin_cluster).
+ --jt <resource manager URL> Only for mapreduce mode: Job submission URL, if not set, taken from ${HADOOP_HOME}/conf.
+ --fs <file system URL> Only for mapreduce mode: Target file system, if not set, taken from ${HADOOP_HOME}/conf.
+ --help Display this help.
+ --verbose Display full command used to start the process.
+ Gobblin Version: 0.15.0
+```
+
+
+1. Standalone:
+ This mode starts all Gobblin services in single JVM on a single node. This mode is useful for development and light weight usage:
+ ```
+ gobblin service standalone start
+ ```
+ For more details and architecture on each execution mode, refer [Standalone-Deployment](/gobblin-docs/user-guide/Gobblin-Deployment.md)
+
+2. Mapreduce:
+
+ This mode is dependent on Hadoop (both MapReduce and HDFS) running locally or remote cluster. Before launching any Gobblin jobs on Hadoop MapReduce, check the Gobblin system configuration file located at `conf/mapreduce/application.properties` for property `fs.uri`, which defines the file system URI used. The default value is `hdfs://localhost:8020`, which points to the local HDFS on the default port 8020. Change it to the right value depending on your Hadoop/HDFS setup. For example, [...]
+``` fs.uri=hdfs://<namenode host name>:9000/ ```
+ * `--jt`: resource manager URL
+ * `--fs`: file system type value for `fs.uri`
+
+ This mode will have the minimum set of Gobblin jars, selected using `libs/gobblin-<module_name>-$GOBBLIN_VERSION.jar`, which is passed as `-libjar` to hadoop command while running the job. These same set of jars also gets added to the Hadoop `DistributedCache` for use in the mappers. If a job has additional jars needed for task executions (in the mappers), those jars can also be included by using the `--jars` option or the following job configuration property in the job configuration file:
+ ```
+ job.jars=<comma-separated list of jars the job depends on>
+ ```
+ if `HADOOP_HOME` is set in the environment, Gobblin will add result of `hadoop classpath` prior to default `GOBBLIN_CLASSPATH` to give them precedence while running `bin/gobblin`.
+
+ All job data and persisted job/task states will be written to the specified file system. Before launching any jobs, make sure the environment variable `HADOOP_HOME` is set so that it can access hadoop binaries under `{HADOOP_HOME}/bin` and also working directory should be set with configuration `{gobblin.cluster.work.dir}`. Note that the Gobblin working directory will be created on the file system specified above.
+
+ An important side effect of this is that (depending on the application) non-fully-qualified paths (like `/my/path`) will default to local file system if `HADOOP_HOME` is not set, while they will default to HDFS if the variable is set. When referring to local paths, it is always a good idea to use the fully qualified path (e.g. `file:///my/path`).
+
+
+3. Cluster Mode (master & worker)
+ This is a cluster mode consist of master and worker process.
+ ```
+ gobblin service cluster-master start
+ gobblin service cluster-worker start
+ ```
+
+4. AWS
+ This mode starts Gobblin on AWS cloud cluster.
+ ```
+ gobblin service aws start
+ ```
+
+5. YARN
+ This mode starts Gobblin on YARN cluster.
+ ```
+ gobblin service yarn start
+ ```
+
+
+Gobblin System Configurations
+----------------------
+
+Following values can be override by setting it in `gobblin-env.sh`
+
+`GOBBLIN_LOGS` : by default the logs are written to `$GOBBLIN_HOME/logs`, it can be override by setting `GOBBLIN_LOGS`\
+`GOBBLIN_VERSION` : by default gobblin version is set by the build process, it can be override by setting `GOBBLIN_VERSION`\
+
+
+All Gobblin system configurations details can be found here: [Configuration Properties Glossary](user-guide/Configuration-Properties-Glossary).
+
diff --git a/gobblin-docs/user-guide/Gobblin-Deployment.md b/gobblin-docs/user-guide/Gobblin-Deployment.md
index 00b8aef..85acfc1 100644
--- a/gobblin-docs/user-guide/Gobblin-Deployment.md
+++ b/gobblin-docs/user-guide/Gobblin-Deployment.md
@@ -3,12 +3,15 @@ Table of Contents
[TOC]
-Deployment Overview <a name="Standalone-Overview"></a>
---------------------
+Gobblin Execution Modes Overview <a name="gobblin-execution-modes-Overview"></a>
+--------------------------------------------------------------------------------
One important feature of Gobblin is that it can be run on different platforms. Currently, Gobblin can run in standalone mode (which runs on a single machine), and on Hadoop MapReduce mode (which runs on a Hadoop cluster). This page summarizes the different deployment modes of Gobblin. It is important to understand the architecture of Gobblin in a specific deployment mode, so this page also describes the architecture of each deployment mode.
Gobblin supports Java 7 and up, but can only run on Hadoop 2.x. By default, Gobblin will build against Hadoop 2.x, run `./gradlew clean build`. More information on how to build Gobblin can be found [here](https://github.com/apache/incubator-gobblin/blob/master/README.md). All directories/paths referred below are relative to `gobblin-dist`.
+To run gobblin in any of the following executuon mode using ```gobblin.sh```, refer [Gobblin-CLI](/gobblin-docs/user-guide/Gobblin-CLI.md) for the usage.
+
+
Standalone Architecture <a name="Standalone-Architecture"></a>
--------------------
The following diagram illustrates the Gobblin standalone architecture. In the standalone mode, a Gobblin instance runs in a single JVM and tasks run in a thread pool, the size of which is configurable. The standalone mode is good for light-weight data sources such as small databases. The standalone mode is also the default mode for trying and testing Gobblin.
@@ -25,73 +28,8 @@ Each `LocalJobLauncher` starts and manages a few components for executing tasks
4. Waiting for all the submitted tasks to finish.
5. Upon completion of all the submitted tasks, collecting tasks states and persisting them to the state store, and publishing the extracted data.
-Standalone Deployment <a name="Standalone-Deployment"></a>
---------------------
-Gobblin ships with a script `bin/gobblin-standalone.sh` for starting and stopping the standalone Gobblin daemon on a single node. Below is the usage of this launch script:
-
-```
-gobblin-standalone.sh <start | status | restart | stop> [OPTION]
-Where:
- --workdir <job work dir> Gobblin's base work directory: if not set, taken from ${GOBBLIN_WORK_DIR}
- --jars <comma-separated list of job jars> Job jar(s): if not set, lib is examined
- --conf <directory of job configuration files> Directory of job configuration files: if not set, taken from
- --help Display this help and exit
-```
-
-In the standalone mode, the `JobScheduler`, upon startup, will pick up job configuration files from a user-defined directory and schedule the jobs to run. The job configuration file directory can be specified using the `--conf` command-line option of `bin/gobblin-standalone.sh` or through an environment variable named `GOBBLIN_JOB_CONFIG_DIR`. The `--conf` option takes precedence and will take the value of `GOBBLIN_JOB_CONFIG_DIR` if not set. Note that this job configuration directory is [...]
-
-The `JobScheduler` is backed by a [Quartz](http://quartz-scheduler.org/) scheduler and it supports cron-based triggers using the configuration property `job.schedule` for defining the cron schedule. Please refer to this [tutorial](http://quartz-scheduler.org/documentation/quartz-2.2.x/tutorials/tutorial-lesson-06) for more information on how to use and configure a cron-based trigger.
-
-Gobblin needs a working directory at runtime, which can be specified using the command-line option `--workdir` of `bin/gobblin-standalone.sh` or an environment variable named `GOBBLIN_WORK_DIR`. The `--workdir` option takes precedence and will take the value of `GOBBLIN_WORK_DIR` if not set. Once started, Gobblin will create some subdirectories under the root working directory, as follows:
-```
-GOBBLIN_WORK_DIR\
- task-staging\ # Staging area where data pulled by individual tasks lands
- task-output\ # Output area where data pulled by individual tasks lands
- job-output\ # Final output area of data pulled by jobs
- state-store\ # Persisted job/task state store
- metrics\ # Metrics store (in the form of metric log files), one subdirectory per job.
-```
-
-Before starting the Gobblin standalone daemon, make sure the environment variable `JAVA_HOME` is properly set to point to the home directory of the Java Runtime Environment (JRE) of choice. When starting the JVM process of the Gobblin standalone daemon, a default set of jars will be included on the `classpath`. Additional jars needed by your Gobblin jobs can be specified as a comma-separated list using the command-line option `--jars` of `bin/gobblin-standaline.sh`. If the `--jar` option [...]
-
-Below is a summary of the environment variables that may be set for standalone deployment.
-
-* `GOBBLIN_JOB_CONFIG_DIR`: this variable defines the directory where job configuration files are stored.
-* `GOBBLIN_WORK_DIR`: this variable defines the working directory for Gobblin to operate.
-* `JAVA_HOME`: this variable defines the path to the home directory of the Java Runtime Environment (JRE) used to run the daemon process.
-
-To start the Gobblin standalone daemon, run the following command:
-```
-bin/gobblin-standalone.sh start [OPTION]
-```
-After the Gobblin standalone daemon is started, the logs can be found under `logs`. Gobblin uses [SLF4J](http://www.slf4j.org/) and the [slf4j-log4j12](http://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12) binding for logging. The [log4j](http://logging.apache.org/log4j/1.2/) configuration can be found at `conf/log4j-standalone.xml`.
-
-By default, the Gobblin standalone daemon uses the following JVM settings. Change the settings in `bin/gobblin-standalone.sh` if necessary for your deployment.
-
-```
--Xmx2g -Xms1g
--XX:+UseConcMarkSweepGC -XX:+UseParNewGC
--XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintTenuringDistribution
--XX:+UseCompressedOops
--XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=<gobblin log dir>
-```
-
-To restart the Gobblin standalone daemon, run the following command:
-```
-bin/gobblin-standalone.sh restart [OPTION]
-```
-
-To stop the running Gobblin standalone daemon, run the following command:
-```
-bin/gobblin-standalone.sh stop
-```
-
-If there are any additional jars that any jobs depend on, the jars can be added to the classpath using the `--jars` option.
-
-The script also supports checking the status of the running daemon process using the `bin/gobblin-standalone.sh status` command.
-
-Hadoop MapReduce Architecture <a name="Hadoop-MapReduce-Architecture"></a>
+MapReduce architecture <a name="MapReduce-Architecture"></a>
--------------------
The digram below shows the architecture of Gobblin on Hadoop MapReduce. As the diagram shows, a Gobblin job runs as a mapper-only MapReduce job that runs tasks of the Gobblin job in the mappers. The basic idea here is to use the mappers purely as _containers_ to run Gobblin tasks. This design also makes it easier to integrate with Yarn. Unlike in the standalone mode, task retries are not handled by Gobblin itself in the Hadoop MapReduce mode. Instead, Gobblin relies on the task retry mec [...]
@@ -118,41 +56,33 @@ A mapper in a Gobblin MapReduce job follows the step below to run tasks assigned
5. Upon completion of all the submitted tasks, writing out the state of each task into a file that will be read by the `MRJobLauncher` when collecting task states.
6. Going back to step 2 and reading the next input record if available.
-Hadoop MapReduce Deployment <a name="Hadoop-MapReduce-Deployment"></a>
---------------------
-Gobblin out-of-the-box ships with a script `bin/gobblin-mapreduce.sh` for launching a Gobblin job on Hadoop MapReduce. Below is the usage of this launch script:
+Master-Worker architecture
+----------------------------------
+
+
+
+
+
+AWS architecture
+---------------
+
+
+
+
+
+YARN architecture
+---------------
+
+
-```
-Usage: gobblin-mapreduce.sh [OPTION] --conf <job configuration file>
-Where OPTION can be:
- --jt <job tracker / resource manager URL> Job submission URL: if not set, taken from ${HADOOP_HOME}/conf
- --fs <file system URL> Target file system: if not set, taken from ${HADOOP_HOME}/conf
- --jars <comma-separated list of job jars> Job jar(s): if not set, lib is examined
- --workdir <job work dir> Gobblin's base work directory: if not set, taken from ${GOBBLIN_WORK_DIR}
- --projectversion <version> Gobblin version to be used. If set, overrides the distribution build version
- --logdir <log dir> Gobblin's log directory: if not set, taken from ${GOBBLIN_LOG_DIR} if present. Otherwise ./logs is used
- --help Display this help and exit
-```
-It is assumed that you already have Hadoop (both MapReduce and HDFS) setup and running somewhere. Before launching any Gobblin jobs on Hadoop MapReduce, check the Gobblin system configuration file located at `conf/gobblin-mapreduce.properties` for property `fs.uri`, which defines the file system URI used. The default value is `hdfs://localhost:8020`, which points to the local HDFS on the default port 8020. Change it to the right value depending on your Hadoop/HDFS setup. For example, if [...]
-```
-fs.uri=hdfs://<namenode host name>:9000/
-```
-Note that if the option `--fs` of `bin/gobblin-mapreduce.sh` is set, the value of `--fs` should be consistent with the value of `fs.uri`.
+Gobblin-As-A-Service architecture
+----------------------------------
-All job data and persisted job/task states will be written to the specified file system. Before launching any jobs, make sure the environment variable `HADOOP_BIN_DIR` is set to point to the `bin` directory under the Hadoop installation directory. Similarly to the standalone deployment, the Hadoop MapReduce deployment also needs a working directory, which can be specified using the command-line option `--workdir` of `bin/gobblin-mapreduce.sh` or the environment variable `GOBBLIN_WORK_DIR [...]
-* `GOBBLIN_WORK_DIR`: this variable defines the working directory for Gobblin to operate.
-* `HADOOP_BIN_DIR`: this variable defines the path to the `bin` directory under the Hadoop installation directory.
-This setup will have the minimum set of jars Gobblin needs to run the job added to the Hadoop `DistributedCache` for use in the mappers. If a job has additional jars needed for task executions (in the mappers), those jars can also be included by using the `--jars` option of `bin/gobblin-mapreduce.sh` or the following job configuration property in the job configuration file:
-```
-job.jars=<comma-separated list of jars the job depends on>
-```
-The `--projectversion` controls which version of the Gobblin jars to look for. Typically, this value is dynamically set during the build process. Users should use the `bin/gobblin-mapreduce.sh` script that is copied into the `gobblin-distribution-[project-version].tar.gz` file. This version of the script has the project version already set, in which case users do not need to specify the `--projectversion` parameter. If users want to use the `gobblin/bin/gobblin-mapreduce.sh` script they [...]
-The `--logdir` parameter controls the directory where log files are written to. If not set log files are written under a the `./logs` directory.
diff --git a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/util/DatabaseJobHistoryStoreSchemaManager.java b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/util/DatabaseJobHistoryStoreSchemaManager.java
index 57f8027..ba1aa08 100644
--- a/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/util/DatabaseJobHistoryStoreSchemaManager.java
+++ b/gobblin-metastore/src/main/java/org/apache/gobblin/metastore/util/DatabaseJobHistoryStoreSchemaManager.java
@@ -25,12 +25,15 @@ import java.util.Properties;
import com.google.common.io.Closer;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.configuration.AbstractConfiguration;
import org.apache.commons.configuration.CompositeConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.PropertiesConfiguration;
import org.apache.commons.configuration.SystemConfiguration;
import org.apache.commons.lang.StringUtils;
+import org.apache.gobblin.annotation.Alias;
+import org.apache.gobblin.runtime.cli.CliApplication;
import org.flywaydb.core.api.FlywayException;
import org.flywaydb.core.api.MigrationInfoService;
import org.flywaydb.core.Flyway;
@@ -44,7 +47,9 @@ import org.apache.gobblin.metastore.DatabaseJobHistoryStore;
*
* @author Joel Baranick
*/
-public class DatabaseJobHistoryStoreSchemaManager implements Closeable {
+@Slf4j
+@Alias(value = "job-store-schema-manager", description = "Database job history store schema manager")
+public class DatabaseJobHistoryStoreSchemaManager implements CliApplication, Closeable {
private final Flyway flyway;
@@ -75,7 +80,8 @@ public class DatabaseJobHistoryStoreSchemaManager implements Closeable {
public void close() throws IOException {
}
- public static void main(String[] args) throws IOException {
+ @Override
+ public void run(String[] args) throws Exception {
if (args.length < 1 || args.length > 2) {
printUsage();
}
diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/cli/CliEmbeddedGobblin.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/cli/CliEmbeddedGobblin.java
index a49562f..929031e 100644
--- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/cli/CliEmbeddedGobblin.java
+++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/cli/CliEmbeddedGobblin.java
@@ -111,7 +111,7 @@ public class CliEmbeddedGobblin implements CliApplication {
private void listQuickApps() {
List<Alias> aliases = getAllAliases();
- System.out.println("Usage: gobblin run <quick-app-name> [OPTIONS]");
+ System.out.println("Usage: gobblin cli run <quick-app-name> [OPTIONS]");
System.out.println("Available quick apps:");
for (Alias thisAlias : aliases) {
System.out.println(String.format("\t%s\t-\t%s", thisAlias.value(), thisAlias.description()));
diff --git a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
index 71a66db..8288923 100644
--- a/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
+++ b/gobblin-runtime/src/main/java/org/apache/gobblin/runtime/util/JobStateToJsonConverter.java
@@ -26,6 +26,7 @@ import java.io.Writer;
import java.util.List;
import java.util.Properties;
+import lombok.extern.slf4j.Slf4j;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.DefaultParser;
@@ -34,6 +35,8 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.lang.StringUtils;
+import org.apache.gobblin.annotation.Alias;
+import org.apache.gobblin.runtime.cli.CliApplication;
import org.apache.hadoop.conf.Configuration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -54,7 +57,9 @@ import org.apache.gobblin.util.JobConfigurationUtils;
*
* @author Yinan Li
*/
-public class JobStateToJsonConverter {
+@Slf4j
+@Alias(value = "job-state-to-json", description = "To convert Job state to JSON")
+public class JobStateToJsonConverter implements CliApplication {
private static final Logger LOGGER = LoggerFactory.getLogger(JobStateToJsonConverter.class);
@@ -154,7 +159,8 @@ public class JobStateToJsonConverter {
}
@SuppressWarnings("all")
- public static void main(String[] args) throws Exception {
+ @Override
+ public void run(String[] args) throws Exception {
Option sysConfigOption = Option.builder("sc").argName("system configuration file")
.desc("Gobblin system configuration file (required if no state store URL specified)").longOpt("sysconfig").hasArg().build();
Option storeUrlOption = Option.builder("u").argName("gobblin state store URL")
@@ -186,13 +192,13 @@ public class JobStateToJsonConverter {
} catch (ParseException pe) {
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("JobStateToJsonConverter", options);
- System.exit(1);
+ return;
}
if (!cmd.hasOption(sysConfigOption.getLongOpt()) && !cmd.hasOption(storeUrlOption.getLongOpt()) ){
HelpFormatter formatter = new HelpFormatter();
formatter.printHelp("JobStateToJsonConverter", options);
- System.exit(1);
+ return;
}
Properties sysConfig = new Properties();
diff --git a/gobblin-utility/src/main/java/org/apache/gobblin/runtime/cli/GobblinCli.java b/gobblin-utility/src/main/java/org/apache/gobblin/runtime/cli/GobblinCli.java
index e1e1a5c..aa3cbd7 100644
--- a/gobblin-utility/src/main/java/org/apache/gobblin/runtime/cli/GobblinCli.java
+++ b/gobblin-utility/src/main/java/org/apache/gobblin/runtime/cli/GobblinCli.java
@@ -22,6 +22,9 @@ import com.google.common.collect.Sets;
import org.apache.gobblin.annotation.Alias;
import org.apache.gobblin.util.ClassAliasResolver;
+import java.util.ArrayList;
+import java.util.List;
+
/**
* Instantiates a {@link CliApplication} and runs it.
@@ -51,10 +54,10 @@ public class GobblinCli {
}
private static void printUsage(ClassAliasResolver<CliApplication> resolver) {
- System.out.println("Usage: gobblin <command>");
+ System.out.println("Usage: gobblin cli <command>");
System.out.println("Available commands:");
for (Alias alias : resolver.getAliasObjects()) {
- System.out.println("\t" + alias.value() + "\t" + alias.description());
+ System.out.println("\t" + alias.value() + "\t\t" + alias.description());
}
}