You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by le...@apache.org on 2017/07/04 23:56:23 UTC
[16/20] incubator-spot git commit: Adding resp codes to Proxy Edge
with lookup on IANA resp code table
Adding resp codes to Proxy Edge with lookup on IANA resp code table
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/a9b66c44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/a9b66c44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/a9b66c44
Branch: refs/heads/SPOT-35_graphql_api
Commit: a9b66c44efda4f4e76c43b3c7034bc25ffc329d8
Parents: bbab834
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Mon Jul 3 21:42:00 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/migrate_old_proxy_data.py | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/a9b66c44/spot-setup/migration/migrate_old_proxy_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.py b/spot-setup/migration/migrate_old_proxy_data.py
index 06ef5f2..61f2e98 100755
--- a/spot-setup/migration/migrate_old_proxy_data.py
+++ b/spot-setup/migration/migrate_old_proxy_data.py
@@ -131,6 +131,14 @@ def main():
util.execute_hive_cmd(insert_cmd, log)
+ ## proxy IANA resp codes
+ log.info("Uploading Proxy IANA resp codes")
+ iana_table_name = 'proxy_iana_rcode_tmp'
+ iana_code_csv = '{0}/oa/components/iana/http-rcode.csv'.format(old_oa_path)
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(iana_code_csv, staging_db, iana_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+
## proxy Edge
log.info("Processing Proxy Edge")
staging_table_name = 'proxy_edge_tmp'
@@ -144,7 +152,7 @@ def main():
load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
util.execute_hive_cmd(load_cmd, log)
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, time, clientip, host, webcat, '0', reqmethod, useragent, resconttype, referer, uriport, serverip, scbytes, csbytes, fulluri, hour(time), respcode FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT e.tdate, e.time, e.clientip, e.host, e.webcat, COALESCE(i.respcode,'0'), e.reqmethod, e.useragent, e.resconttype, e.referer, e.uriport, e.serverip, e.scbytes, e.csbytes, e.fulluri, hour(e.time), e.respcode FROM {5}.{6} e left join {5}.{7} i on e.respcode = i.respcode_name ;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name, iana_table_name)
util.execute_hive_cmd(insert_cmd, log)
@@ -165,13 +173,6 @@ def main():
##proxy Timeline
log.info("Processing Proxy Timeline")
-
- log.info("Uploading Proxy IANA resp codes")
- iana_table_name = 'proxy_iana_rcode_tmp'
- iana_code_csv = '{0}/oa/components/iana/http-rcode.csv'.format(old_oa_path)
- load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(iana_code_csv, staging_db, iana_table_name)
- util.execute_hive_cmd(load_cmd, log)
-
staging_table_name = 'proxy_timeline_tmp'
dest_table_name = 'proxy_timeline'
pattern = 'timeline*.tsv'
@@ -195,7 +196,7 @@ def main():
util.execute_hive_cmd(load_cmd, log)
# Insert into new table from staging table and adding FullURI value
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', t.tstart, t.tend, t.duration, t.clientip, t.respcode, i.respcode_name FROM {6}.{7} t left join {6}.{8} i on t.respcode = i.respcode where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name, iana_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', t.tstart, t.tend, t.duration, t.clientip, t.respcode, COALESCE(i.respcode_name,'') FROM {6}.{7} t left join {6}.{8} i on t.respcode = i.respcode where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name, iana_table_name)
util.execute_hive_cmd(insert_cmd, log)
else: