You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by ev...@apache.org on 2017/03/29 16:52:01 UTC
[37/50] [abbrv] incubator-spot git commit: Adding permissions to
Impala and fixing Proxy table partitions
Adding permissions to Impala and fixing Proxy table partitions
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/bb8847ac
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/bb8847ac
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/bb8847ac
Branch: refs/heads/SPOT-35_graphql_api
Commit: bb8847acb06df80d0d231df21331537bf6c2749c
Parents: 5af8419
Author: Moises Valdovinos <mv...@mvaldovi-mac01.amr.corp.intel.com>
Authored: Thu Mar 9 18:37:37 2017 -0600
Committer: Diego Ortiz Huerta <di...@intel.com>
Committed: Wed Mar 15 11:50:20 2017 -0700
----------------------------------------------------------------------
spot-setup/create_dns_avro_parquet.hql | 37 ------
spot-setup/create_dns_parquet.hql | 147 +++++++++++++++++++++
spot-setup/create_flow_avro_parquet.hql | 72 -----------
spot-setup/create_flow_parquet.hql | 179 ++++++++++++++++++++++++++
spot-setup/create_proxy_avro_parquet.hql | 76 -----------
spot-setup/create_proxy_parquet.hql | 161 +++++++++++++++++++++++
spot-setup/hdfs_setup.sh | 46 ++++---
7 files changed, 518 insertions(+), 200 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bb8847ac/spot-setup/create_dns_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_dns_avro_parquet.hql b/spot-setup/create_dns_avro_parquet.hql
deleted file mode 100755
index 105c386..0000000
--- a/spot-setup/create_dns_avro_parquet.hql
+++ /dev/null
@@ -1,37 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns (
- frame_time STRING,
- unix_tstamp BIGINT,
- frame_len INT,
- ip_dst STRING,
- ip_src STRING,
- dns_qry_name STRING,
- dns_qry_class STRING,
- dns_qry_type INT,
- dns_qry_rcode INT,
- dns_a STRING
-)
-PARTITIONED BY (y INT, m INT, d INT, h int)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/dns/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "DnsRecord"
- , "namespace" : "com.cloudera.accelerators.dns.avro"
- , "fields": [
- {"name": "frame_time", "type":["string", "null"]}
- , {"name": "unix_tstamp", "type":["bigint", "null"]}
- , {"name": "frame_len", "type":["int", "null"]}
- , {"name": "ip_dst", "type":["string", "null"]}
- , {"name": "ip_src", "type":["string", "null"]}
- , {"name": "dns_qry_name", "type":["string", "null"]}
- , {"name": "dns_qry_class", "type":["string", "null"]}
- , {"name": "dns_qry_type", "type":["int", "null"]}
- , {"name": "dns_qry_rcode", "type":["int", "null"]}
- , {"name": "dns_a", "type":["string", "null"]}
- ]
-}');
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bb8847ac/spot-setup/create_dns_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_dns_parquet.hql b/spot-setup/create_dns_parquet.hql
new file mode 100755
index 0000000..4e01d26
--- /dev/null
+++ b/spot-setup/create_dns_parquet.hql
@@ -0,0 +1,147 @@
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns (
+frame_time STRING,
+unix_tstamp BIGINT,
+frame_len INT,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+dns_a STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT,
+h TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro (
+unix_tstamp BIGINT,
+dns_a STRING,
+dns_qry_name STRING,
+ip_dst STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/dendro';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_edge (
+unix_tstamp BIGINT,
+frame_len BIGINT,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+dns_a STRING,
+hh INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_scores (
+frame_time STRING,
+unix_tstamp BIGINT,
+frame_len BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+ml_score FLOAT,
+tld STRING,
+query_rep STRING,
+hh INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard (
+ip_threat STRING,
+dns_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro (
+anchor STRING,
+total BIGINT,
+dns_qry_name STRING,
+ip_dst STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/threat_dendro';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_investigation (
+unix_tstamp BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+ip_sev INT,
+dns_sev INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/threat_investigation';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bb8847ac/spot-setup/create_flow_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_flow_avro_parquet.hql b/spot-setup/create_flow_avro_parquet.hql
deleted file mode 100755
index 34aa63b..0000000
--- a/spot-setup/create_flow_avro_parquet.hql
+++ /dev/null
@@ -1,72 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow (
- treceived STRING,
- unix_tstamp BIGINT,
- tryear INT,
- trmonth INT,
- trday INT,
- trhour INT,
- trminute INT,
- trsec INT,
- tdur FLOAT,
- sip STRING,
- dip STRING,
- sport INT,
- dport INT,
- proto STRING,
- flag STRING,
- fwd INT,
- stos INT,
- ipkt BIGINT,
- ibyt BIGINT,
- opkt BIGINT,
- obyt BIGINT,
- input INT,
- output INT,
- sas INT,
- das INT,
- dtos INT,
- dir INT,
- rip STRING
- )
-PARTITIONED BY (y INT, m INT, d INT, h int)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/flow/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "FlowRecord"
- , "namespace" : "com.cloudera.accelerators.flows.avro"
- , "fields": [
- {"name": "treceived", "type":["string", "null"]}
- , {"name": "unix_tstamp", "type":["long", "null"]}
- , {"name": "tryear", "type":["int", "null"]}
- , {"name": "trmonth", "type":["int", "null"]}
- , {"name": "trday", "type":["int", "null"]}
- , {"name": "trhour", "type":["int", "null"]}
- , {"name": "trminute", "type":["int", "null"]}
- , {"name": "trsec", "type":["int", "null"]}
- , {"name": "tdur", "type":["float", "null"]}
- , {"name": "sip", "type":["string", "null"]}
- , {"name": "sport", "type":["int", "null"]}
- , {"name": "dip", "type":["string", "null"]}
- , {"name": "dport", "type":["int", "null"]}
- , {"name": "proto", "type":["string", "null"]}
- , {"name": "flag", "type":["string", "null"]}
- , {"name": "fwd", "type":["int", "null"]}
- , {"name": "stos", "type":["int", "null"]}
- , {"name": "ipkt", "type":["bigint", "null"]}
- , {"name": "ibytt", "type":["bigint", "null"]}
- , {"name": "opkt", "type":["bigint", "null"]}
- , {"name": "obyt", "type":["bigint", "null"]}
- , {"name": "input", "type":["int", "null"]}
- , {"name": "output", "type":["int", "null"]}
- , {"name": "sas", "type":["int", "null"]}
- , {"name": "das", "type":["int", "null"]}
- , {"name": "dtos", "type":["int", "null"]}
- , {"name": "dir", "type":["int", "null"]}
- , {"name": "rip", "type":["string", "null"]}
- ]
-}');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bb8847ac/spot-setup/create_flow_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_flow_parquet.hql b/spot-setup/create_flow_parquet.hql
new file mode 100755
index 0000000..c1da27f
--- /dev/null
+++ b/spot-setup/create_flow_parquet.hql
@@ -0,0 +1,179 @@
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow (
+treceived STRING,
+unix_tstamp BIGINT,
+tryear INT,
+trmonth INT,
+trday INT,
+trhour INT,
+trminute INT,
+trsec INT,
+tdur FLOAT,
+sip STRING,
+dip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flag STRING,
+fwd INT,
+stos INT,
+ipkt BIGINT,
+ibyt BIGINT,
+opkt BIGINT,
+obyt BIGINT,
+input INT,
+output INT,
+sas INT,
+das INT,
+dtos INT,
+dir INT,
+rip STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT,
+h TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_chords (
+ip_threat STRING,
+srcip STRING,
+dstip STRING,
+ibyt BIGINT,
+ipkt BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/chords';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_edge (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flags STRING,
+tos INT,
+ibyt BIGINT,
+ipkt BIGINT,
+input BIGINT,
+output BIGINT,
+rip STRING,
+obyt BIGINT,
+opkt BIGINT,
+hh INT,
+mn INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_scores (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+ipkt INT,
+ibyt INT,
+opkt INT,
+obyt INT,
+ml_score FLOAT,
+rank INT,
+srcip_INTernal INT,
+dstip_INTernal INT,
+src_geoloc STRING,
+dst_geoloc STRING,
+src_domain STRING,
+dst_domain STRING,
+src_rep STRING,
+dst_rep STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard (
+ip_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_threat_investigation (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+srcport INT,
+dstport INT,
+score INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/threat_investigation';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline (
+ip_threat STRING,
+tstart STRING,
+tend STRING,
+srcip STRING,
+dstip STRING,
+proto STRING,
+sport INT,
+dport INT,
+ipkt BIGINT,
+ibyt BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/timeline';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bb8847ac/spot-setup/create_proxy_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_proxy_avro_parquet.hql b/spot-setup/create_proxy_avro_parquet.hql
deleted file mode 100755
index 3fd7ea8..0000000
--- a/spot-setup/create_proxy_avro_parquet.hql
+++ /dev/null
@@ -1,76 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy (
-p_date string,
-p_time string,
-clientip string,
-host string,
-reqmethod string,
-useragent string,
-resconttype string,
-duration int,
-username string,
-authgroup string,
-exceptionid string,
-filterresult string,
-webcat string,
-referer string,
-respcode string,
-action string,
-urischeme string,
-uriport string,
-uripath string,
-uriquery string,
-uriextension string,
-serverip string,
-scbytes int,
-csbytes int,
-virusid string,
-bcappname string,
-bcappoper string,
-fulluri string
-)
-PARTITIONED BY (
-y string,
-m string,
-d string,
-h string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/proxy/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "ProxyRecord"
- , "namespace" : "com.cloudera.accelerators.proxy.avro"
- , "fields": [
- {"name": "p_date", "type":["string", "null"]}
- , {"name": "p_time", "type":["string", "null"]}
- , {"name": "clientip", "type":["string", "null"]}
- , {"name": "host", "type":["string", "null"]}
- , {"name": "reqmethod", "type":["string", "null"]}
- , {"name": "useragent", "type":["string", "null"]}
- , {"name": "resconttype", "type":["string", "null"]}
- , {"name": "duration", "type":["int", "null"]}
- , {"name": "username", "type":["string", "null"]}
- , {"name": "authgroup", "type":["string", "null"]}
- , {"name": "exceptionid", "type":["string", "null"]}
- , {"name": "filterresult", "type":["string", "null"]}
- , {"name": "webcat", "type":["string", "null"]}
- , {"name": "referer", "type":["string", "null"]}
- , {"name": "respcode", "type":["string", "null"]}
- , {"name": "action", "type":["string", "null"]}
- , {"name": "urischeme", "type":["string", "null"]}
- , {"name": "uriport", "type":["string", "null"]}
- , {"name": "uripath", "type":["string", "null"]}
- , {"name": "uriquery", "type":["string", "null"]}
- , {"name": "uriextension", "type":["string", "null"]}
- , {"name": "serverip", "type":["string", "null"]}
- , {"name": "scbytes", "type":["int", "null"]}
- , {"name": "csbytes", "type":["int", "null"]}
- , {"name": "virusid", "type":["string", "null"]}
- , {"name": "bcappname", "type":["string", "null"]}
- , {"name": "bcappoper", "type":["string", "null"]}
- , {"name": "fulluri", "type":["string", "null"]}
- ]
-}');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bb8847ac/spot-setup/create_proxy_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_proxy_parquet.hql b/spot-setup/create_proxy_parquet.hql
new file mode 100755
index 0000000..f28b831
--- /dev/null
+++ b/spot-setup/create_proxy_parquet.hql
@@ -0,0 +1,161 @@
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy (
+p_date STRING,
+p_time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+authgroup STRING,
+exceptionid STRING,
+filterresult STRING,
+webcat STRING,
+referer STRING,
+respcode STRING,
+action STRING,
+urischeme STRING,
+uriport STRING,
+uripath STRING,
+uriquery STRING,
+uriextension STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+virusid STRING,
+bcappname STRING,
+bcappoper STRING,
+fulluri STRING
+)
+PARTITIONED BY (
+y STRING,
+m STRING,
+d STRING,
+h STRING
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+webcat STRING,
+respcode STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+referer STRING,
+uriport STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+hh INT,
+respcode_name STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+webcat STRING,
+referer STRING,
+respcode INT,
+uriport INT,
+uripath STRING,
+uriquery STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+word STRING,
+ml_score FLOAT,
+uri_rep STRING,
+respcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard (
+p_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_threat_investigation (
+tdate STRING,
+fulluri STRING,
+uri_sev INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/threat_investigation';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline (
+p_threat STRING,
+tstart STRING,
+tend STRING,
+duration BIGINT,
+clientip STRING,
+respcode STRING,
+respcodename STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/timeline';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bb8847ac/spot-setup/hdfs_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/hdfs_setup.sh b/spot-setup/hdfs_setup.sh
index 8eb9ae3..9c557ba 100755
--- a/spot-setup/hdfs_setup.sh
+++ b/spot-setup/hdfs_setup.sh
@@ -18,35 +18,51 @@
#
DSOURCES=('flow' 'dns' 'proxy')
-DFOLDERS=('binary' 'hive' 'stage')
+DFOLDERS=('binary'
+'stage'
+'hive'
+'hive/oa'
+'hive/oa/chords'
+'hive/oa/edge'
+'hive/oa/summary'
+'hive/oa/suspicious'
+'hive/oa/storyboard'
+'hive/oa/threat_investigation'
+'hive/oa/timeline'
+'hive/oa/dendro'
+'hive/oa/threat_dendro'
+)
+
+# Sourcing spot configuration variables
source /etc/spot.conf
-#
-# creating HDFS user's folder
-#
-hadoop fs -mkdir ${HUSER}
-hadoop fs -chown ${USER}:supergroup ${HUSER}
+# Creating HDFS user's folder
+sudo -u hdfs hdfs dfs -mkdir ${HUSER}
+sudo -u hdfs hdfs dfs -chown ${USER}:supergroup ${HUSER}
+sudo -u hdfs hdfs dfs -chmod 775 ${HUSER}
+# Creating HDFS paths for each use case
for d in "${DSOURCES[@]}"
do
echo "creating /$d"
- hadoop fs -mkdir ${HUSER}/$d
+ hdfs dfs -mkdir ${HUSER}/$d
for f in "${DFOLDERS[@]}"
do
echo "creating $d/$f"
- hadoop fs -mkdir ${HUSER}/$d/$f
+ hdfs dfs -mkdir ${HUSER}/$d/$f
done
done
-#
-# create hive tables
-#
-#configure / create catalog
-hive -e "CREATE DATABASE ${DBNAME}"
+# Modifying permission on HDFS folders to allow Impala to read/write
+hdfs dfs -chmod -R 775 ${HUSER}
+hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}
+
+# Creating Spot Database
+impala-shell -i ${IMPALA_DEM} -q "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
+# Creating Impala tables
for d in "${DSOURCES[@]}"
do
- hive -hiveconf huser=${HUSER} -hiveconf dbname=${DBNAME} -f create_${d}_avro_parquet.hql
+ impala-shell -i ${IMPALA_DEM} --var=huser=${HUSER} --var=dbname=${DBNAME} -f create_${d}_parquet.hql
done
-