You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@fluo.apache.org by GitBox <gi...@apache.org> on 2018/06/06 21:19:31 UTC
[GitHub] mikewalch closed pull request #181: Minimize output 128
mikewalch closed pull request #181: Minimize output 128
URL: https://github.com/apache/fluo-uno/pull/181
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/README.md b/README.md
index e89b9c3..d290902 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ setup a Fluo instance.
git clone https://github.com/apache/fluo-uno.git
cd fluo-uno
./bin/uno fetch accumulo # Fetches binary tarballs of Accumulo and its dependencies
-./bin/uno setup accumulo # Sets up Accumulo and its dependencies (Hadoop & Zookeeper)
+./bin/uno setup accumulo # Sets up Accumulo and its dependencies (Hadoop & ZooKeeper)
eval "$(./bin/uno env)" # Bash-specific command that sets up current shell
```
@@ -74,7 +74,7 @@ vim conf/uno.conf
All commands are run using the `uno` script in `bin/`. Uno has a command that helps you configure
your shell so that you can run commands from any directory and easily set common environment
-variables in your shell for Uno, Hadoop, Zookeeper, Fluo, and Spark. Run the following command to
+variables in your shell for Uno, Hadoop, ZooKeeper, Fluo, and Spark. Run the following command to
print this shell configuration. You can also add `--paths` or `--vars` to the command below to limit
output to PATH or environment variable configuration:
@@ -106,7 +106,7 @@ upgrade components and need to download/build the latest version.
The `setup` command will install the downloaded tarballs to the directory set by `$INSTALL` in your
`uno.conf` and run you local development cluster. The command can be run in several different ways:
-1. Sets up Apache Accumulo and its dependencies of Hadoop, Zookeeper. This starts all processes and
+1. Sets up Apache Accumulo and its dependencies of Hadoop, ZooKeeper. This starts all processes and
will wipe Accumulo/Hadoop if this command was run previously.
uno setup accumulo
@@ -130,7 +130,7 @@ The `setup` command will install the downloaded tarballs to the directory set by
uno setup spark
-6. Sets up all components (Fluo, Accumulo, Hadoop, Zookeeper, Spark, metrics service).
+6. Sets up all components (Fluo, Accumulo, Hadoop, ZooKeeper, Spark, metrics service).
uno setup all
diff --git a/bin/impl/fetch.sh b/bin/impl/fetch.sh
index 980543a..7163095 100755
--- a/bin/impl/fetch.sh
+++ b/bin/impl/fetch.sh
@@ -159,8 +159,8 @@ metrics)
echo "Usage: uno fetch <component>"
echo -e "\nPossible components:\n"
echo " all Fetches all binary tarballs of the following components"
- echo " accumulo Downloads Accumulo, Hadoop & Zookeeper. Builds Accumulo if repo set in uno.conf"
- echo " fluo Downloads Fluo, Accumulo, Hadoop & Zookeeper. Builds Fluo or Accumulo if repo set in uno.conf"
+ echo " accumulo Downloads Accumulo, Hadoop & ZooKeeper. Builds Accumulo if repo set in uno.conf"
+ echo " fluo Downloads Fluo, Accumulo, Hadoop & ZooKeeper. Builds Fluo or Accumulo if repo set in uno.conf"
echo " metrics Downloads InfluxDB and Grafana"
echo " spark Downloads Spark"
echo "Options:"
diff --git a/bin/impl/setup-accumulo.sh b/bin/impl/setup-accumulo.sh
index 174f52b..0d5ea2f 100755
--- a/bin/impl/setup-accumulo.sh
+++ b/bin/impl/setup-accumulo.sh
@@ -21,8 +21,8 @@ if [[ -z "$ACCUMULO_REPO" ]]; then
fi
if [[ $1 != "--no-deps" ]]; then
- "$UNO_HOME"/bin/impl/setup-hadoop.sh
- "$UNO_HOME"/bin/impl/setup-zookeeper.sh
+ run_setup_script Hadoop
+ run_setup_script ZooKeeper
fi
pkill -f accumulo.start
@@ -30,7 +30,7 @@ pkill -f accumulo.start
# stop if any command fails
set -e
-echo "Setting up Apache Accumulo at $ACCUMULO_HOME"
+echo >&0 "Setting up Apache Accumulo at $ACCUMULO_HOME"
rm -rf "$INSTALL"/accumulo-*
rm -f "$ACCUMULO_LOG_DIR"/*
@@ -78,7 +78,7 @@ if [[ "$1" == "--with-metrics" ]]; then
echo "accumulo.sink.graphite.server_port=2004"
echo "accumulo.sink.graphite.metrics_prefix=accumulo"
} >> "$conf"/"$metrics_props"
- "$UNO_HOME"/bin/impl/setup-metrics.sh
+ run_setup_script Metrics
fi
if [[ "$ACCUMULO_USE_NATIVE_MAP" == "true" ]]; then
@@ -98,4 +98,3 @@ else
"$ACCUMULO_HOME"/bin/accumulo-cluster start
fi
-echo "Apache Accumulo setup complete"
diff --git a/bin/impl/setup-fluo-yarn.sh b/bin/impl/setup-fluo-yarn.sh
index ae896d3..ddcbafb 100755
--- a/bin/impl/setup-fluo-yarn.sh
+++ b/bin/impl/setup-fluo-yarn.sh
@@ -24,11 +24,11 @@ if [[ -z "$FLUO_YARN_REPO" ]]; then
fi
if [[ $1 != "--no-deps" ]]; then
- "$UNO_HOME"/bin/impl/setup-fluo.sh
+ run_setup_script Fluo
fi
if [[ -f "$DOWNLOADS/$FLUO_YARN_TARBALL" ]]; then
- echo "Setting up Apache Fluo YARN launcher at $FLUO_YARN_HOME"
+ echo >&0 "Setting up Apache Fluo YARN launcher at $FLUO_YARN_HOME"
# Don't stop if pkills fail
set +e
pkill -f "fluo\.yarn"
@@ -51,10 +51,8 @@ if [[ -f "$DOWNLOADS/$FLUO_YARN_TARBALL" ]]; then
"$FLUO_YARN_HOME"/lib/fetch.sh
- echo "Apache Fluo YARN launcher setup complete"
-
stty sane
else
- echo "WARNING: Apache Fluo YARN launcher tarball '$FLUO_YARN_TARBALL' was not found in $DOWNLOADS."
- echo "Apache Fluo YARN launcher will not be set up!"
+ echo >&0 "WARNING: Apache Fluo YARN launcher tarball '$FLUO_YARN_TARBALL' was not found in $DOWNLOADS."
+ echo >&0 "Apache Fluo YARN launcher will not be set up!"
fi
diff --git a/bin/impl/setup-fluo.sh b/bin/impl/setup-fluo.sh
index b1ee442..424a74d 100755
--- a/bin/impl/setup-fluo.sh
+++ b/bin/impl/setup-fluo.sh
@@ -24,11 +24,11 @@ if [[ -z "$FLUO_REPO" ]]; then
fi
if [[ $1 != "--no-deps" ]]; then
- "$UNO_HOME"/bin/impl/setup-accumulo.sh
+ run_setup_script Accumulo
fi
if [[ -f "$DOWNLOADS/$FLUO_TARBALL" ]]; then
- echo "Setting up Apache Fluo at $FLUO_HOME"
+ echo >&0 "Setting up Apache Fluo at $FLUO_HOME"
# Don't stop if pkills fail
set +e
pkill -f fluo.yarn
@@ -70,10 +70,8 @@ if [[ -f "$DOWNLOADS/$FLUO_TARBALL" ]]; then
"$FLUO_HOME"/lib/fetch.sh extra
- echo "Apache Fluo setup complete"
-
stty sane
else
- echo "WARNING: Apache Fluo tarball '$FLUO_TARBALL' was not found in $DOWNLOADS."
- echo "Apache Fluo will not be set up!"
+ echo >&0 "WARNING: Apache Fluo tarball '$FLUO_TARBALL' was not found in $DOWNLOADS."
+ echo >&0 "Apache Fluo will not be set up!"
fi
diff --git a/bin/impl/setup-hadoop.sh b/bin/impl/setup-hadoop.sh
index d2bf0e0..d8a81ff 100755
--- a/bin/impl/setup-hadoop.sh
+++ b/bin/impl/setup-hadoop.sh
@@ -24,7 +24,7 @@ pkill -f hadoop.yarn
# stop if any command fails
set -e
-echo "Setting up Apache Hadoop at $HADOOP_PREFIX"
+echo >&0 "Setting up Apache Hadoop at $HADOOP_PREFIX"
rm -rf "$INSTALL"/hadoop-*
rm -f "$HADOOP_LOG_DIR"/*
@@ -51,4 +51,3 @@ $SED "s#YARN_LOG_DIR=[^ ]*#YARN_LOG_DIR=$YARN_LOG_DIR#g" "$hadoop_conf/yarn-env.
"$HADOOP_PREFIX"/sbin/start-dfs.sh
"$HADOOP_PREFIX"/sbin/start-yarn.sh
-echo "Apache Hadoop setup complete"
diff --git a/bin/impl/setup-metrics.sh b/bin/impl/setup-metrics.sh
index f4b3c45..e045ca1 100755
--- a/bin/impl/setup-metrics.sh
+++ b/bin/impl/setup-metrics.sh
@@ -17,11 +17,11 @@
source "$UNO_HOME"/bin/impl/util.sh
if [[ "$OSTYPE" == "darwin"* ]]; then
- echo "The metrics services (InfluxDB and Grafana) are not supported on Mac OS X at this time."
+ echo >&0 "The metrics services (InfluxDB and Grafana) are not supported on Mac OS X at this time."
exit 1
fi
-echo "Killing InfluxDB & Grafana (if running)"
+echo >&0 "Killing InfluxDB & Grafana (if running)"
pkill -f influxdb
pkill -f grafana-server
@@ -35,36 +35,36 @@ verify_exist_hash "$GRAFANA_TARBALL" "$GRAFANA_HASH"
INFLUXDB_TARBALL=influxdb-"$INFLUXDB_VERSION".tar.gz
GRAFANA_TARBALL=grafana-"$GRAFANA_VERSION".tar.gz
if [[ ! -f "$DOWNLOADS/build/$INFLUXDB_TARBALL" ]]; then
- echo "InfluxDB tarball $INFLUXDB_TARBALL does not exists in downloads/build/"
+ echo >&0 "InfluxDB tarball $INFLUXDB_TARBALL does not exists in downloads/build/"
exit 1
fi
if [[ ! -f "$DOWNLOADS/build/$GRAFANA_TARBALL" ]]; then
- echo "Grafana tarball $GRAFANA_TARBALL does not exists in downloads/build"
+ echo >&0 "Grafana tarball $GRAFANA_TARBALL does not exists in downloads/build"
exit 1
fi
if [[ ! -d "$FLUO_HOME" ]]; then
- echo "Fluo must be installed before setting up metrics"
+ echo >&0 "Fluo must be installed before setting up metrics"
exit 1
fi
# stop if any command fails
set -e
-echo "Removing previous versions of InfluxDB & Grafana"
+echo >&0 "Removing previous versions of InfluxDB & Grafana"
rm -rf "$INSTALL"/influxdb-*
rm -rf "$INSTALL"/grafana-*
-echo "Remove previous log and data dirs"
+echo >&0 "Remove previous log and data dirs"
rm -f "$LOGS_DIR"/metrics/*
rm -rf "$DATA_DIR"/influxdb
mkdir -p "$LOGS_DIR"/metrics
-echo "Setting up metrics (influxdb + grafana)..."
+echo >&0 "Setting up metrics (influxdb + grafana)..."
tar xzf "$DOWNLOADS/build/$INFLUXDB_TARBALL" -C "$INSTALL"
"$INFLUXDB_HOME"/bin/influxd config -config "$UNO_HOME"/conf/influxdb/influxdb.conf > "$INFLUXDB_HOME"/influxdb.conf
if [[ ! -f "$INFLUXDB_HOME"/influxdb.conf ]]; then
- echo "Failed to create $INFLUXDB_HOME/influxdb.conf"
+ echo >&0 "Failed to create $INFLUXDB_HOME/influxdb.conf"
exit 1
fi
$SED "s#DATA_DIR#$DATA_DIR#g" "$INFLUXDB_HOME"/influxdb.conf
@@ -79,7 +79,7 @@ cp "$FLUO_HOME"/contrib/grafana/* "$GRAFANA_HOME"/dashboards/
cp "$UNO_HOME"/conf/grafana/accumulo-dashboard.json "$GRAFANA_HOME"/dashboards/
"$GRAFANA_HOME"/bin/grafana-server -homepath="$GRAFANA_HOME" 2> /dev/null &
-echo "Configuring Fluo to send metrics to InfluxDB"
+echo >&0 "Configuring Fluo to send metrics to InfluxDB"
if [[ $FLUO_VERSION =~ ^1\.[0-1].*$ ]]; then
FLUO_PROPS=$FLUO_HOME/conf/fluo.properties
else
@@ -94,14 +94,14 @@ $SED "/fluo.metrics.reporter.graphite/d" "$FLUO_PROPS"
echo "fluo.metrics.reporter.graphite.frequency=30"
} >> "$FLUO_PROPS"
-echo "Configuring InfluxDB..."
+echo >&0 "Configuring InfluxDB..."
sleep 10
"$INFLUXDB_HOME"/bin/influx -import -path "$FLUO_HOME"/contrib/influxdb/fluo_metrics_setup.txt
# allow commands to fail
set +e
-echo "Configuring Grafana..."
+echo >&0 "Configuring Grafana..."
sleep 5
@@ -112,11 +112,11 @@ function add_datasource() {
--data-binary "$1"
retcode=$?
if [[ $retcode != 0 ]]; then
- echo "Failed to add Grafana data source. Retrying in 5 sec.."
+ echo >&0 "Failed to add Grafana data source. Retrying in 5 sec.."
sleep 5
fi
done
- echo ""
+ echo >&0 ""
}
accumulo_data='{"name":"accumulo_metrics","type":"influxdb","url":"http://'
diff --git a/bin/impl/setup-spark.sh b/bin/impl/setup-spark.sh
index c7688c4..3f015f9 100755
--- a/bin/impl/setup-spark.sh
+++ b/bin/impl/setup-spark.sh
@@ -19,11 +19,11 @@ source "$UNO_HOME"/bin/impl/util.sh
verify_exist_hash "$SPARK_TARBALL" "$SPARK_HASH"
if [[ ! -d "$HADOOP_PREFIX" ]]; then
- echo "Apache Hadoop needs to be setup before Apache Spark can be setup."
+ echo >&0 "Apache Hadoop needs to be setup before Apache Spark can be setup."
exit 1
fi
-echo "Setting up Apache Spark at $SPARK_HOME"
+echo >&0 "Setting up Apache Spark at $SPARK_HOME"
pkill -f org.apache.spark.deploy.history.HistoryServer
@@ -45,4 +45,3 @@ $SED "s#LOGS_DIR#$LOGS_DIR#g" "$SPARK_HOME"/conf/spark-defaults.conf
export SPARK_LOG_DIR=$LOGS_DIR/spark
"$SPARK_HOME"/sbin/start-history-server.sh
-echo "Apache Spark setup complete"
diff --git a/bin/impl/setup-zookeeper.sh b/bin/impl/setup-zookeeper.sh
index 919e0ff..9a6ef37 100755
--- a/bin/impl/setup-zookeeper.sh
+++ b/bin/impl/setup-zookeeper.sh
@@ -23,7 +23,7 @@ pkill -f QuorumPeerMain
# stop if any command fails
set -e
-echo "Setting up Apache Zookeeper at $ZOOKEEPER_HOME"
+echo >&0 "Setting up Apache ZooKeeper at $ZOOKEEPER_HOME"
rm -rf "$INSTALL"/zookeeper-*
rm -f "$ZOO_LOG_DIR"/*
mkdir -p "$ZOO_LOG_DIR"
@@ -36,4 +36,3 @@ $SED "s#DATA_DIR#$DATA_DIR#g" "$ZOOKEEPER_HOME"/conf/zoo.cfg
rm -rf "$DATA_DIR"/zookeeper
"$ZOOKEEPER_HOME"/bin/zkServer.sh start
-echo "Apache Zookeeper setup complete"
diff --git a/bin/impl/start.sh b/bin/impl/start.sh
index 4e437c2..8fe8b00 100755
--- a/bin/impl/start.sh
+++ b/bin/impl/start.sh
@@ -26,7 +26,7 @@ case "$1" in
tmp="$(pgrep -f QuorumPeerMain | tr '\n' ' ')"
if [[ -z "$tmp" ]]; then
"$ZOOKEEPER_HOME"/bin/zkServer.sh start
- else echo "Zookeeper already running at: $tmp"
+ else echo "ZooKeeper already running at: $tmp"
fi
tmp="$(pgrep -f hadoop\\.hdfs | tr '\n' ' ')"
@@ -73,7 +73,7 @@ case "$1" in
tmp="$(pgrep -f QuorumPeerMain | tr '\n' ' ')"
if [[ -z "$tmp" ]]; then
"$ZOOKEEPER_HOME"/bin/zkServer.sh start
- else echo "Zookeeper already running at: $tmp"
+ else echo "ZooKeeper already running at: $tmp"
fi
;;
metrics)
@@ -100,9 +100,9 @@ case "$1" in
*)
echo "Usage: uno start <component> [--no-deps]"
echo -e "\nPossible components:\n"
- echo " accumulo Start Apache Accumulo plus dependencies: Hadoop, Zookeeper"
+ echo " accumulo Start Apache Accumulo plus dependencies: Hadoop, ZooKeeper"
echo " hadoop Start Apache Hadoop"
- echo " zookeeper Start Apache Zookeeper"
+ echo " zookeeper Start Apache ZooKeeper"
echo " metrics Start InfluxDB and Grafana"
echo "Options:"
echo " --no-deps Dependencies will start unless this option is specified. Only works for accumulo component."
diff --git a/bin/impl/stop.sh b/bin/impl/stop.sh
index 030d083..617e70c 100755
--- a/bin/impl/stop.sh
+++ b/bin/impl/stop.sh
@@ -77,9 +77,9 @@ case "$1" in
*)
echo "Usage: uno stop <component> [--no-deps]"
echo -e "\nPossible components:\n"
- echo " accumulo Stop Apache Accumulo plus dependencies: Hadoop, Zookeeper"
+ echo " accumulo Stop Apache Accumulo plus dependencies: Hadoop, ZooKeeper"
echo " hadoop Stop Apache Hadoop"
- echo " zookeeper Stop Apache Zookeeper"
+ echo " zookeeper Stop Apache ZooKeeper"
echo "Options:"
echo " --no-deps Dependencies will stop unless this option is specified. Only works for accumulo component."
exit 1
diff --git a/bin/impl/util.sh b/bin/impl/util.sh
index 14bbe7c..ac1a4fa 100755
--- a/bin/impl/util.sh
+++ b/bin/impl/util.sh
@@ -19,7 +19,7 @@ function verify_exist_hash() {
expected_hash=$(echo "${2// /}" | tr '[:upper:]' '[:lower:]')
if [[ ! -f "$DOWNLOADS/$tarball" ]]; then
- echo "The tarball $tarball does not exist in downloads/"
+ echo >&0 "The tarball $tarball does not exist in downloads/"
exit 1
fi
@@ -30,14 +30,14 @@ function verify_exist_hash() {
64) HASH_CMD='shasum -a 256' ;;
128) HASH_CMD='shasum -a 512' ;;
*)
- echo "Expected checksum ($expected_hash) of $tarball is not an MD5, SHA1, SHA256, or SHA512 sum"
+ echo >&0 "Expected checksum ($expected_hash) of $tarball is not an MD5, SHA1, SHA256, or SHA512 sum"
exit 1
;;
esac
actual_hash=$($HASH_CMD "$DOWNLOADS/$tarball" | awk '{print $1}')
if [[ "$actual_hash" != "$expected_hash" ]]; then
- echo "The actual checksum ($actual_hash) of $tarball does not match the expected checksum ($expected_hash)"
+ echo >&0 "The actual checksum ($actual_hash) of $tarball does not match the expected checksum ($expected_hash)"
exit 1
fi
}
@@ -46,8 +46,16 @@ function verify_exist_hash() {
function check_dirs() {
for arg in "$@"; do
if [[ ! -d "${!arg}" ]]; then
- echo "$arg=${!arg} is not a valid directory. Please make sure it exists"
+ echo >&0 "$arg=${!arg} is not a valid directory. Please make sure it exists"
exit 1
fi
done
}
+
+function run_setup_script() {
+ local SCRIP; SCRIP=$(echo "$1" | tr '[:upper:] ' '[:lower:]-')
+ local L_DIR; L_DIR="$LOGS_DIR/setup"
+ mkdir -p "$L_DIR"
+ shift
+ "$UNO_HOME/bin/impl/setup-$SCRIP.sh" "$@" 1>"$L_DIR/$SCRIP.stdout" 2>"$L_DIR/$SCRIP.stderr"
+}
diff --git a/bin/uno b/bin/uno
index 9500573..94c74ab 100755
--- a/bin/uno
+++ b/bin/uno
@@ -40,41 +40,49 @@ fetch)
fi
;;
setup)
+ [[ -n $LOGS_DIR ]] && rm -f "$LOGS_DIR"/setup/*.std{out,err}
+ echo "Beginning setup (detailed logs in $LOGS_DIR/setup)..."
case "$2" in
all)
- "$bin"/impl/setup-fluo.sh
- "$bin"/impl/setup-spark.sh
- "$bin"/impl/setup-metrics.sh
+ run_setup_script Fluo
+ run_setup_script Spark
+ run_setup_script Metrics
;;
accumulo)
- "$bin"/impl/setup-accumulo.sh "$3"
+ run_setup_script Accumulo "$3"
;;
fluo)
- "$bin"/impl/setup-fluo.sh "$3"
+ run_setup_script Fluo "$3"
;;
fluo-yarn)
- "$bin"/impl/setup-fluo-yarn.sh "$3"
+ run_setup_script "Fluo Yarn" "$3"
;;
spark)
- "$bin"/impl/setup-spark.sh
+ run_setup_script Spark
;;
metrics)
- "$bin"/impl/setup-metrics.sh
+ run_setup_script Metrics
;;
*)
echo "Usage: uno setup <component> [--no-deps]"
echo -e "\nPossible components:\n"
echo " all Sets up all of the following components"
- echo " accumulo Sets up Apache Accumulo and its dependencies (Hadoop & Zookeeper)"
+ echo " accumulo Sets up Apache Accumulo and its dependencies (Hadoop & ZooKeeper)"
echo " spark Sets up Apache Spark"
- echo " fluo Sets up Apache Fluo and its dependencies (Accumulo, Hadoop, & Zookeeper)"
- echo " fluo-yarn Sets up Apache Fluo YARN and its dependencies (Fluo, Accumulo, Hadoop, & Zookeeper)"
+ echo " fluo Sets up Apache Fluo and its dependencies (Accumulo, Hadoop, & ZooKeeper)"
+ echo " fluo-yarn Sets up Apache Fluo YARN and its dependencies (Fluo, Accumulo, Hadoop, & ZooKeeper)"
echo -e " metrics Sets up metrics service (InfluxDB + Grafana)\n"
echo "Options:"
echo " --no-deps Dependencies will be setup unless this option is specified. Only works for fluo & accumulo components."
exit 1
;;
esac
+ if [[ "$?" == 0 ]]; then
+ echo "Setup complete."
+ else
+ echo "Setup failed!"
+ false
+ fi
;;
kill)
"$bin"/impl/kill.sh "${@:2}"
@@ -103,8 +111,8 @@ version)
echo " Run 'uno fetch' for a list of possible components."
echo " setup <component> Sets up component and its dependencies (clearing any existing data)"
echo " Run 'uno setup' for list of components."
- echo " start <component> Start Zookeeper, Hadoop, Accumulo, if not running."
- echo " stop <component> Stop Accumulo, Hadoop, Zookeeper, if running."
+ echo " start <component> Start ZooKeeper, Hadoop, Accumulo, if not running."
+ echo " stop <component> Stop Accumulo, Hadoop, ZooKeeper, if running."
echo " kill Kills all software"
echo " ashell Runs the Accumulo shell"
echo " env Prints out shell configuration for PATH and common environment variables."
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services