You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/03/19 19:09:24 UTC
[13/15] incubator-spot git commit: Set user hdfs cmd for putting
files on hdfs
Set user hdfs cmd for putting files on hdfs
In the case where we're building ODM tables to be stored as avro, we need to be able to store the schemas on the hdfs. This requires us to specify and run an hdfs command with the user that owns the odm setup files locally. The user_hdfs_cmd allows us to reference the user which is running this script, but also likely owns the avro schema files locally.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/2ae343d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/2ae343d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/2ae343d9
Branch: refs/heads/SPOT-181_ODM
Commit: 2ae343d921f3b0b64cfe70951dd14dcdecd3c2f2
Parents: 8fa024b
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Wed Feb 14 14:00:57 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Wed Feb 14 14:00:57 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/2ae343d9/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 38731de..1927f0e 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -100,7 +100,9 @@ source $SPOTCONF
# Check no-sudo argument and set the proper hdfs command to run our create table statements later
if [[ ${no_sudo} == "true" ]]; then
hdfs_cmd="hdfs"
+ user_hdfs_cmd="hdfs"
+ # If HADOOP_USER_NAME already set, don't attempt to set as hdfs
if [[ ! -z "${HADOOP_USER_NAME}" ]]; then
log "HADOOP_USER_NAME: ${HADOOP_USER_NAME}"
else
@@ -109,6 +111,7 @@ if [[ ${no_sudo} == "true" ]]; then
fi
else
hdfs_cmd="sudo -u hdfs hdfs"
+ user_hdfs_cmd="sudo -u ${USER} hdfs"
fi
# Creating HDFS user's folder
@@ -178,9 +181,9 @@ do
if [ "$format" == "avro" ] ; then
log "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
- log "${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
+ log "${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
- ${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
+ ${user_hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
log "Creating ODM Impala Avro table ${f}..."
log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"