You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2018/03/19 19:09:19 UTC
[08/15] incubator-spot git commit: Update ODM table creation process
Update ODM table creation process
- Using common logger to show progress through the table creation steps.
- Our constructed impala shell with arguments is also being passed in to ensure all of the proper flags and configuration are set when running these commands.
- Added hdfs_cmd to the step where we put the avro schemas on the hdfs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/c0f38704
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/c0f38704
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/c0f38704
Branch: refs/heads/SPOT-181_ODM
Commit: c0f38704593c02ec704a921cb5170fac1c98598f
Parents: aa09bfb
Author: Tadd Wood <ta...@arcadiadata.com>
Authored: Mon Feb 12 10:42:32 2018 -0800
Committer: Tadd Wood <ta...@arcadiadata.com>
Committed: Mon Feb 12 10:49:07 2018 -0800
----------------------------------------------------------------------
spot-setup/odm/odm_setup.sh | 28 ++++++++++++++++------------
1 file changed, 16 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/c0f38704/spot-setup/odm/odm_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/odm/odm_setup.sh b/spot-setup/odm/odm_setup.sh
index 100e3bb..3abefc8 100755
--- a/spot-setup/odm/odm_setup.sh
+++ b/spot-setup/odm/odm_setup.sh
@@ -152,28 +152,32 @@ log "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
${impala_db_shell} "CREATE DATABASE IF NOT EXISTS ${DBNAME}";
# Creating ODM Impala tables
+
for d in "${DSOURCES[@]}"
do
for f in "${DFOLDERS[@]}"
do
- #If desired storage format is parquet, create ODM as Parquet tables
+ # If desired storage format is parquet, create ODM as Parquet tables
+
if [ "$format" == "pqt" ] ; then
- echo "Creating ODM Impala Parquet table ${f}..."
- echo "impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql"
-
- impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql
+ log "Creating ODM Impala Parquet table ${f}..."
+ log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql"
+
+ ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} -c -f create_${f}_pqt.sql
fi
- # If desired storage format is "avro", create ODM as Avro tables with Avro schemas
+
+ # If desired storage format is avro, create ODM as Avro tables with Avro schemas
+
if [ "$format" == "avro" ] ; then
- echo "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
- echo "sudo -u ${USER} hdfs dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
+ log "Adding ${f} Avro schema to ${HUSER}/$d/schema ..."
+ log "${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc"
- sudo -u ${USER} hdfs dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
+ ${hdfs_cmd} dfs -put -f $f.avsc ${HUSER}/$d/schema/$f.avsc
- echo "Creating ODM Impala Avro table ${f}..."
- echo "impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
+ log "Creating ODM Impala Avro table ${f}..."
+ log "${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql"
- impala-shell -i ${IMPALA_DEM} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql
+ ${impala_db_shell} --var=ODM_DBNAME=${DBNAME} --var=ODM_TABLENAME=${f} --var=ODM_LOCATION=${HUSER}/${d}/${f} --var=ODM_AVRO_URL=hdfs://${HUSER}/${d}/schema/${f}.avsc -c -f create_${f}_avro.sql
fi
done
done
\ No newline at end of file