You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/03/19 19:09:12 UTC
[01/15] incubator-spot git commit: Add logger and directory checker
for ODM Setup
Repository: incubator-spot
Updated Branches:
refs/heads/SPOT-181_ODM 97291e90d -> 0e3ef34a0
Add logger and directory checker for ODM Setup
These changes were pulled from SPOT-213. The generalized logger should help reduce some clutter with echos in the script, and the safe_mkdir() should help run the directory setup process more elegantly by not trying to recreate directories or throw warnings when directories already exist.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/4e990c4d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/4e990c4d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/4e990c4d
Branch: refs/heads/SPOT-181_ODM
Commit: 4e990c4df73905a15dc5a19e4209dff9b26a06c9
Parents: 97291e9
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:06:47 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:47:51 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 22 ++++++++++++++++++++++
1 file changed, 22 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4e990c4d/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index a2d8a51..534998c 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -24,6 +24,28 @@
#
# NOTE: At this time only Parquet and Avro storage formats are supported for the ODM tables.
+
+set -e
+
+function log() {
+ # General logger for the ODM setup script that prints any input provided to it
+ printf "hdfs_setup.sh:\n $1\n"
+}
+
+function safe_mkdir() {
+ # 1. Takes the hdfs command options and a directory
+ # 2. Checks for the directory before trying to create it and keeps the script from creating existing directories
+
+ local hdfs_cmd=$1
+ local dir=$2
+ if $(hdfs dfs -test -d ${dir}); then
+ log "${dir} already exists"
+ else
+ log "running mkdir on ${dir}"
+ ${hdfs_cmd} dfs -mkdir ${dir}
+ fi
+}
+
# Check the format argument and make sure its supported
format=$1
if [ "$format" != "pqt" ] && [ "$format" != "avro" ] ; then
[03/15] incubator-spot git commit: Add argument parser and handle new
arguments
Posted by na...@apache.org.
Add argument parser and handle new arguments
- Pulled most of these changes from SPOT-213
- Also moved the format argument to be captured within the "-f" flag
- Format checking is then performed after we've parsed all of the input arguments.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/17f39597
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/17f39597
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/17f39597
Branch: refs/heads/SPOT-181_ODM
Commit: 17f395973c3554ebbe5023b4e455be28c8dbca25
Parents: 7f96905
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:15:51 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:48:33 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 35 ++++++++++++++++++++++++++++-------
1 file changed, 28 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/17f39597/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 9c8eb9d..f23d2e3 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -46,13 +46,6 @@ function safe_mkdir() {
fi
}
-# Check the format argument and make sure its supported
-format=$1
-if [ "$format" != "pqt" ] && [ "$format" != "avro" ] ; then
- echo "Format argument '$format' is not supported. Only Parquet and Avro are supported data storage formats. Use 'pqt' or 'avro' instead (i.e. ./odm_setup pqt)."
- exit 0
-fi
-
SPOTCONF="/etc/spot.conf"
DSOURCES=('odm')
DFOLDERS=(
@@ -64,6 +57,34 @@ DFOLDERS=(
'vulnerability_context'
)
+# Check input argument options
+for arg in "$@"; do
+ case $arg in
+ "--no-sudo")
+ log "not using sudo"
+ no_sudo=true
+ shift
+ ;;
+ "-c")
+ shift
+ SPOTCONF=$1
+ log "Spot Configuration file: ${SPOTCONF}"
+ shift
+ ;;
+ "-f")
+ shift
+ format=$1
+ shift
+ ;;
+ esac
+done
+
+# Check the format argument and make sure its supported
+if [ "$format" != "pqt" ] && [ "$format" != "avro" ] ; then
+ log "Format argument '$format' is not supported. Only Parquet and Avro are supported data storage formats. Use 'pqt' or 'avro' instead (i.e. ./odm_setup pqt)."
+ exit 1
+fi
+
# Sourcing ODM Spot configuration variables
source /etc/spot.conf
[07/15] incubator-spot git commit: Handle Kerberos with impala shell
Posted by na...@apache.org.
Handle Kerberos with impala shell
- Set a common impala shell configuration that can be shared during the execution of CREATE DATABASE and CREATE TABLE commands.
- Check if Kerberos is enabled, and if so modify the common impala shell configuration to include the -k flag
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/aa09bfb3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/aa09bfb3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/aa09bfb3
Branch: refs/heads/SPOT-181_ODM
Commit: aa09bfb36e6c22e1092508b0408254eb16bad9ef
Parents: 4998b22
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:33:49 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:49:00 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/aa09bfb3/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index ce87f29..100e3bb 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -134,8 +134,22 @@ do
${hdfs_cmd} dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
done
+# Check if Kerberos is enabled, and create the proper impala-shell configuration and arguments to be used when creating the ODM tables
+
+log "Using Impala as execution engine."
+impala_db_shell="impala-shell -i ${IMPALA_DEM}"
+log "${impala_db_shell}"
+
+if [[ ${KERBEROS} == "true" ]]; then
+ log "Kerberos enabled. Modifying Impala Shell arguments"
+ impala_db_shell="${impala_db_shell} -k"
+ log "${impala_db_shell}"
+fi
+
# Creating Spot Database
-impala-shell -i ${IMPALA_DEM} -q "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
+
+log "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
+${impala_db_shell} "CREATE DATABASE IF NOT EXISTS ${DBNAME}";
# Creating ODM Impala tables
for d in "${DSOURCES[@]}"
[06/15] incubator-spot git commit: Use safe_mkdir() when creating
HDFS directories and modifying permissions
Posted by na...@apache.org.
Use safe_mkdir() when creating HDFS directories and modifying permissions
- Pulled most of these changes from SPOT-213
- Also updated the echo statements to use log() instead
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/4998b22a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/4998b22a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/4998b22a
Branch: refs/heads/SPOT-181_ODM
Commit: 4998b22ad4b09e2b8d0c493a17e235f0b9776e99
Parents: c83230d
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:26:38 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:48:53 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 27 ++++++++++++++-------------
1 file changed, 14 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/4998b22a/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 4f93d05..ce87f29 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -104,33 +104,34 @@ else
fi
# Creating HDFS user's folder
-sudo -u hdfs hdfs dfs -mkdir ${HUSER}
-sudo -u hdfs hdfs dfs -chown ${USER}:supergroup ${HUSER}
-sudo -u hdfs hdfs dfs -chmod 775 ${HUSER}
+log "creating ${HUSER}"
+safe_mkdir ${hdfs_cmd} ${HUSER}
+${hdfs_cmd} dfs -chown ${USER}:supergroup ${HUSER}
+${hdfs_cmd} dfs -chmod 775 ${HUSER}
# Creating HDFS paths for each use case
for d in "${DSOURCES[@]}"
do
- echo "creating /$d"
- sudo -u hdfs hdfs dfs -mkdir ${HUSER}/$d
+ log "creating /$d"
+ safe_mkdir hdfs ${HUSER}/$d
# Create Avro schemas directory on HDFS if Avro storage is selected
if [ "$format" == "avro" ] ; then
- echo "creating /$d/schema"
- sudo -u hdfs hdfs dfs -mkdir ${HUSER}/$d/schema
+ log "creating ${HUSER}/$d/schema"
+ safe_mkdir ${hdfs_cmd} ${HUSER}/$d/schema
fi
for f in "${DFOLDERS[@]}"
do
- echo "creating $d/$f"
- sudo -u hdfs hdfs dfs -mkdir ${HUSER}/$d/$f
+ log "creating ${HUSER}/$d/$f"
+ safe_mkdir ${hdfs_cmd} ${HUSER}/$d/$f
done
# Modifying permission on HDFS folders to allow Impala to read/write
- echo "modifying permissions recursively on ${HUSER}/$d"
- sudo -u hdfs hdfs dfs -chmod -R 775 ${HUSER}/$d
- sudo -u hdfs hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
- sudo -u hdfs hdfs dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
+ log "modifying permissions recursively on ${HUSER}/$d"
+ hdfs dfs -chmod -R 775 ${HUSER}/$d
+ ${hdfs_cmd} dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
+ ${hdfs_cmd} dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
done
# Creating Spot Database
[10/15] incubator-spot git commit: Add quotes around safe_dir
arguments so that they are safely handled
Posted by na...@apache.org.
Add quotes around safe_dir arguments so that they are safely handled
Without quotes, each argument passed in will get parsed in pieces if they have spaces.
(i.e. sudo -u hdfs hdfs dfs ---> $1=sudo and $2=-u when passed into safe_dir() )
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/1bbb8087
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/1bbb8087
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/1bbb8087
Branch: refs/heads/SPOT-181_ODM
Commit: 1bbb80877112acbf46cc569fcb3a7c18f31ef3ee
Parents: 095695a
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 12:32:32 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 12:32:32 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/1bbb8087/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index dd31d55..8de7f7e 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -44,8 +44,8 @@ function safe_mkdir() {
# 1. Takes the hdfs command options and a directory
# 2. Checks for the directory before trying to create it and keeps the script from creating existing directories
- local hdfs_cmd=$1
- local dir=$2
+ local hdfs_cmd="$1"
+ local dir="$2"
if $(hdfs dfs -test -d ${dir}); then
log "${dir} already exists"
else
@@ -113,7 +113,7 @@ fi
# Creating HDFS user's folder
log "creating ${HUSER}"
-safe_mkdir ${hdfs_cmd} ${HUSER}
+safe_mkdir "${hdfs_cmd}" "${HUSER}"
${hdfs_cmd} dfs -chown ${USER}:supergroup ${HUSER}
${hdfs_cmd} dfs -chmod 775 ${HUSER}
@@ -121,18 +121,18 @@ ${hdfs_cmd} dfs -chmod 775 ${HUSER}
for d in "${DSOURCES[@]}"
do
log "creating /$d"
- safe_mkdir hdfs ${HUSER}/$d
+ safe_mkdir "${hdfs_cmd}" "${HUSER}/$d"
# Create Avro schemas directory on HDFS if Avro storage is selected
if [ "$format" == "avro" ] ; then
log "creating ${HUSER}/$d/schema"
- safe_mkdir ${hdfs_cmd} ${HUSER}/$d/schema
+ safe_mkdir "${hdfs_cmd}" "${HUSER}/$d/schema"
fi
for f in "${DFOLDERS[@]}"
do
log "creating ${HUSER}/$d/$f"
- safe_mkdir ${hdfs_cmd} ${HUSER}/$d/$f
+ safe_mkdir "${hdfs_cmd}" "${HUSER}/$d/$f"
done
# Modifying permission on HDFS folders to allow Impala to read/write
[02/15] incubator-spot git commit: Add env variable config for
Spot.conf
Posted by na...@apache.org.
Add env variable config for Spot.conf
Pulled this change from SPOT-213.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/7f969056
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/7f969056
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/7f969056
Branch: refs/heads/SPOT-181_ODM
Commit: 7f9690564c13c9d541b3bd7e627fa9a41fdb8768
Parents: 4e990c4
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:07:35 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:48:24 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/7f969056/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 534998c..9c8eb9d 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -53,6 +53,7 @@ if [ "$format" != "pqt" ] && [ "$format" != "avro" ] ; then
exit 0
fi
+SPOTCONF="/etc/spot.conf"
DSOURCES=('odm')
DFOLDERS=(
'event'
[04/15] incubator-spot git commit: Change sourcing of spot.conf
Posted by na...@apache.org.
Change sourcing of spot.conf
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/46c0f3f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/46c0f3f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/46c0f3f3
Branch: refs/heads/SPOT-181_ODM
Commit: 46c0f3f316642869b07047f0853ec3de935cc677
Parents: 17f3959
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:16:26 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:48:39 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/46c0f3f3/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index f23d2e3..0aa46f1 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -85,8 +85,9 @@ if [ "$format" != "pqt" ] && [ "$format" != "avro" ] ; then
exit 1
fi
-# Sourcing ODM Spot configuration variables
-source /etc/spot.conf
+# Sourcing spot configuration variables
+log "Sourcing ${SPOTCONF}\n"
+source $SPOTCONF
# Creating HDFS user's folder
sudo -u hdfs hdfs dfs -mkdir ${HUSER}
[05/15] incubator-spot git commit: Check no-sudo argument and set the
command/user we'll run with for each hdfs command
Posted by na...@apache.org.
Check no-sudo argument and set the command/user we'll run with for each hdfs command
- Pulled most of these changes from SPOT-213
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/c83230d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/c83230d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/c83230d4
Branch: refs/heads/SPOT-181_ODM
Commit: c83230d47565d217db10861a4621ad4ddbad0a12
Parents: 46c0f3f
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:20:37 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:48:46 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/c83230d4/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 0aa46f1..4f93d05 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -89,6 +89,20 @@ fi
log "Sourcing ${SPOTCONF}\n"
source $SPOTCONF
+# Check no-sudo argument and set the proper hdfs command to run our create table statements later
+if [[ ${no_sudo} == "true" ]]; then
+ hdfs_cmd="hdfs"
+
+ if [[ ! -z "${HADOOP_USER_NAME}" ]]; then
+ log "HADOOP_USER_NAME: ${HADOOP_USER_NAME}"
+ else
+ log "setting HADOOP_USER_NAME to hdfs"
+ HADOOP_USER_NAME=hdfs
+ fi
+else
+ hdfs_cmd="sudo -u hdfs hdfs"
+fi
+
# Creating HDFS user's folder
sudo -u hdfs hdfs dfs -mkdir ${HUSER}
sudo -u hdfs hdfs dfs -chown ${USER}:supergroup ${HUSER}
[11/15] incubator-spot git commit: Fix rogue hdfs cmd
Posted by na...@apache.org.
Fix rogue hdfs cmd
Missed a replacement of an old hdfs command, which caused permission to not be applied properly after all the directories get created.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/085c2f85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/085c2f85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/085c2f85
Branch: refs/heads/SPOT-181_ODM
Commit: 085c2f855cbfb6ff21d145990515a7c795bc9e94
Parents: 1bbb808
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 12:44:46 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 12:44:46 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/085c2f85/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 8de7f7e..d3165a2 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -137,7 +137,7 @@ do
# Modifying permission on HDFS folders to allow Impala to read/write
log "modifying permissions recursively on ${HUSER}/$d"
- hdfs dfs -chmod -R 775 ${HUSER}/$d
+ ${hdfs_cmd} dfs -chmod -R 775 ${HUSER}/$d
${hdfs_cmd} dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
${hdfs_cmd} dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
done
[14/15] incubator-spot git commit: Set relative path for referencing
ODM files
Posted by na...@apache.org.
Set relative path for referencing ODM files
To make this script more flexible, you should now be able to run it either within the odm directory itself or anywhere outside of it.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/549f492d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/549f492d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/549f492d
Branch: refs/heads/SPOT-181_ODM
Commit: 549f492dcb9205d30b89ba43914c96d8063f6bc2
Parents: 2ae343d
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 14:16:19 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 14:16:19 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 15 +++++++++------
1 file changed, 9 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/549f492d/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 1927f0e..7fc10bd 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -54,6 +54,9 @@ function safe_mkdir() {
fi
}
+# Set path where local files adjacent to odm_setup.sh can be sourced
+ODM_FILES_DIR="$(dirname "$0")"
+
SPOTCONF="/etc/spot.conf"
DSOURCES=('odm')
DFOLDERS=(
@@ -172,23 +175,23 @@ do
if [ "$format" == "pqt" ] ; then
log "Creating ODM Impala Parquet table ${f}..."
- log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql"
+ log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f ${ODM_FILES_DIR}/create_${f}_pqt.sql"
- ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql
+ ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f ${ODM_FILES_DIR}/create_${f}_pqt.sql
fi
# If desired storage format is avro, create ODM as Avro tables with Avro schemas
if [ "$format" == "avro" ] ; then
log "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
- log "${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
+ log "${user_hdfs_cmd} dfs -put -f ${ODM_FILES_DIR}/$f.avsc ${HUSER}/$d/schema/$f.avsc"
- ${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
+ ${user_hdfs_cmd} dfs -put -f ${ODM_FILES_DIR}/$f.avsc ${HUSER}/$d/schema/$f.avsc
log "Creating ODM Impala Avro table ${f}..."
- log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
+ log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f ${ODM_FILES_DIR}/create_${f}_avro.sql"
- ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql
+ ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f ${ODM_FILES_DIR}/create_${f}_avro.sql
fi
done
done
\ No newline at end of file
[15/15] incubator-spot git commit: Update spot.conf
Posted by na...@apache.org.
Update spot.conf
- Updates sourced from SPOT-213
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/0e3ef34a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/0e3ef34a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/0e3ef34a
Branch: refs/heads/SPOT-181_ODM
Commit: 0e3ef34a0098bda834382ed1678471b6314b9507
Parents: 549f492
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 14:23:37 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 14:23:37 2018 -0800
----------------------------------------------------------------------
spot-setup/spot.conf | 34 +++++++++++++++++++++++++---------
1 file changed, 25 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0e3ef34a/spot-setup/spot.conf
----------------------------------------------------------------------
diff --git a/spot-setup/spot.conf b/spot-setup/spot.conf
index 03f221a..407e38f 100755
--- a/spot-setup/spot.conf
+++ b/spot-setup/spot.conf
@@ -15,13 +15,12 @@
# limitations under the License.
-#node configuration
+# Spot node configuration
UINODE='node03'
MLNODE='node04'
GWNODE='node16'
-DBNAME='spot'
-#hdfs - base user and data source config
+# hdfs - base user and data source config
HUSER='/user/spot'
NAME_NODE=''
WEB_PORT=50070
@@ -30,21 +29,38 @@ PROXY_PATH=${HUSER}/${DSOURCE}/hive/y=${YR}/m=${MH}/d=${DY}/
FLOW_PATH=${HUSER}/${DSOURCE}/hive/y=${YR}/m=${MH}/d=${DY}/
HPATH=${HUSER}/${DSOURCE}/scored_results/${FDATE}
-FLOW_TABLE=flow_view
-DNS_TABLE=dns_view
-PROXY_TABLE=proxy_view
+# Database config
+DBNAME='spot'
+DBENGINE=""
-#impala config
+# Impala config
IMPALA_DEM=node04
IMPALA_PORT=21050
-#local fs base user and data source config
+# Kerberos config
+KERBEROS='false'
+KINIT=/usr/bin/kinit
+PRINCIPAL='user'
+KEYTAB='/opt/security/user.keytab'
+SASL_MECH='GSSAPI'
+SECURITY_PROTO='sasl_plaintext'
+KAFKA_SERVICE_NAME=''
+
+# SSL config
+SSL='false'
+SSL_VERIFY='true'
+CA_LOCATION=''
+CERT=''
+KEY=''
+
+# Local fs base user and data source config
LUSER='/home/spot'
LPATH=${LUSER}/ml/${DSOURCE}/${FDATE}
RPATH=${LUSER}/ipython/user/${FDATE}
LIPATH=${LUSER}/ingest
-#dns suspicious connects config
+# DNS suspicious connects config
+
USER_DOMAIN=''
SPK_EXEC=''
[09/15] incubator-spot git commit: Fix logging message to show
odm_setup.sh instead of hdfs_setup
Posted by na...@apache.org.
Fix logging message to show odm_setup.sh instead of hdfs_setup
Also added more notes on how to use certain arguments when running odm_setup.sh at the top of the script.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/095695a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/095695a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/095695a3
Branch: refs/heads/SPOT-181_ODM
Commit: 095695a3943ce0de9acc34293d55988d05d64390
Parents: c0f3870
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 12:02:48 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 12:02:48 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 12 ++++++++++--
1 file changed, 10 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/095695a3/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 3abefc8..dd31d55 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -20,7 +20,15 @@
# Instructions
# To execute this script, run ./odm_setup with a format type (pqt, avro) as an argument.
#
-# i.e. ./odm_setup pqt
+# i.e. ./odm_setup -f pqt
+#
+# Required Arguments:
+# -f : desired storage format for ODM tables
+#
+# Optional Arguments:
+# --no-sudo : run hdfs commands without sudo
+# -c : provide custom path to spot.conf
+#
#
# NOTE: At this time only Parquet and Avro storage formats are supported for the ODM tables.
@@ -29,7 +37,7 @@ set -e
function log() {
# General logger for the ODM setup script that prints any input provided to it
- printf "hdfs_setup.sh:\n $1\n"
+ printf "odm_setup.sh:\n $1\n"
}
function safe_mkdir() {
[08/15] incubator-spot git commit: Update ODM table creation process
Posted by na...@apache.org.
Update ODM table creation process
- Using common logger to show progress through the table creation steps.
- Our constructed impala shell with arguments is also being passed in to ensure all of the proper flags and configuration are set when running these commands.
- Added hdfs_cmd to the step where we put the avro schemas on the hdfs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/c0f38704
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/c0f38704
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/c0f38704
Branch: refs/heads/SPOT-181_ODM
Commit: c0f38704593c02ec704a921cb5170fac1c98598f
Parents: aa09bfb
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:42:32 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:49:07 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 28 ++++++++++++++++------------
1 file changed, 16 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/c0f38704/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 100e3bb..3abefc8 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -152,28 +152,32 @@ log "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
${impala_db_shell} "CREATE DATABASE IF NOT EXISTS ${DBNAME}";
# Creating ODM Impala tables
+
for d in "${DSOURCES[@]}"
do
for f in "${DFOLDERS[@]}"
do
- #If desired storage format is parquet, create ODM as Parquet tables
+ # If desired storage format is parquet, create ODM as Parquet tables
+
if [ "$format" == "pqt" ] ; then
- echo "Creating ODM Impala Parquet table ${f}..."
- echo "impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql"
-
- impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql
+ log "Creating ODM Impala Parquet table ${f}..."
+ log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql"
+
+ ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql
fi
- # If desired storage format is "avro", create ODM as Avro tables with Avro schemas
+
+ # If desired storage format is avro, create ODM as Avro tables with Avro schemas
+
if [ "$format" == "avro" ] ; then
- echo "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
- echo "sudo -u ${USER} hdfs dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
+ log "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
+ log "${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
- sudo -u ${USER} hdfs dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
+ ${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
- echo "Creating ODM Impala Avro table ${f}..."
- echo "impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
+ log "Creating ODM Impala Avro table ${f}..."
+ log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
- impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql
+ ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql
fi
done
done
\ No newline at end of file
[13/15] incubator-spot git commit: Set user hdfs cmd for putting
files on hdfs
Posted by na...@apache.org.
Set user hdfs cmd for putting files on hdfs
In the case where we're building ODM tables to be stored as avro, we need to be able to store the schemas on the hdfs. This requires us to specify and run an hdfs command with the user that owns the odm setup files locally. The user_hdfs_cmd allows us to reference the user which is running this script, but also likely owns the avro schema files locally.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/2ae343d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/2ae343d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/2ae343d9
Branch: refs/heads/SPOT-181_ODM
Commit: 2ae343d921f3b0b64cfe70951dd14dcdecd3c2f2
Parents: 8fa024b
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 14:00:57 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 14:00:57 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/2ae343d9/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 38731de..1927f0e 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -100,7 +100,9 @@ source $SPOTCONF
# Check no-sudo argument and set the proper hdfs command to run our create table statements later
if [[ ${no_sudo} == "true" ]]; then
hdfs_cmd="hdfs"
+ user_hdfs_cmd="hdfs"
+ # If HADOOP_USER_NAME already set, don't attempt to set as hdfs
if [[ ! -z "${HADOOP_USER_NAME}" ]]; then
log "HADOOP_USER_NAME: ${HADOOP_USER_NAME}"
else
@@ -109,6 +111,7 @@ if [[ ${no_sudo} == "true" ]]; then
fi
else
hdfs_cmd="sudo -u hdfs hdfs"
+ user_hdfs_cmd="sudo -u ${USER} hdfs"
fi
# Creating HDFS user's folder
@@ -178,9 +181,9 @@ do
if [ "$format" == "avro" ] ; then
log "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
- log "${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
+ log "${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
- ${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
+ ${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
log "Creating ODM Impala Avro table ${f}..."
log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
[12/15] incubator-spot git commit: Add missing argument for explicit
impala query
Posted by na...@apache.org.
Add missing argument for explicit impala query
Forgot the -q argument in our impala query to create the spot database.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/8fa024b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/8fa024b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/8fa024b7
Branch: refs/heads/SPOT-181_ODM
Commit: 8fa024b7acacfcfa863c0251daf35f60a6cc0860
Parents: 085c2f8
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 12:49:48 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 12:49:48 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/8fa024b7/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index d3165a2..38731de 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -157,7 +157,7 @@ fi
# Creating Spot Database
log "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
-${impala_db_shell} "CREATE DATABASE IF NOT EXISTS ${DBNAME}";
+${impala_db_shell} -q "CREATE DATABASE IF NOT EXISTS ${DBNAME}";
# Creating ODM Impala tables