You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by le...@apache.org on 2017/07/04 23:56:08 UTC
[01/20] incubator-spot git commit: Fixing DNS Threat Dendro shifted
columns
Repository: incubator-spot
Updated Branches:
refs/heads/SPOT-35_graphql_api ffa5b87e6 -> aaabf0774
Fixing DNS Threat Dendro shifted columns
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/aaabf077
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/aaabf077
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/aaabf077
Branch: refs/heads/SPOT-35_graphql_api
Commit: aaabf0774f883c9ae9ea288fefbd0113fc9e1c68
Parents: 020d98c
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Tue Jul 4 17:56:43 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/create_dns_migration_tables.hql | 4 ++--
spot-setup/migration/migrate_old_dns_data.py | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/aaabf077/spot-setup/migration/create_dns_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_dns_migration_tables.hql b/spot-setup/migration/create_dns_migration_tables.hql
index b564486..76e5a4c 100644
--- a/spot-setup/migration/create_dns_migration_tables.hql
+++ b/spot-setup/migration/create_dns_migration_tables.hql
@@ -85,10 +85,10 @@ TBLPROPERTIES ('skip.header.line.count'='1');
DROP TABLE IF EXISTS ${var:dbname}.dns_threat_dendro_tmp;
CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro_tmp (
-anchor STRING,
total BIGINT,
dns_qry_name STRING,
-ip_dst STRING
+ip_dst STRING,
+sev int
)
ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
LOCATION '${var:hpath}/dns/threat_dendro'
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/aaabf077/spot-setup/migration/migrate_old_dns_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.py b/spot-setup/migration/migrate_old_dns_data.py
index 912f79b..fb7dca8 100755
--- a/spot-setup/migration/migrate_old_dns_data.py
+++ b/spot-setup/migration/migrate_old_dns_data.py
@@ -210,7 +210,7 @@ def main():
load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
util.execute_hive_cmd(load_cmd, log)
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', total, dns_qry_name, ip_dst FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', total, dns_qry_name, ip_dst FROM {6}.{7} WHERE dns_qry_name is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
util.execute_hive_cmd(insert_cmd, log)
[05/20] incubator-spot git commit: Fixing shifted fields and table
schema for flow ingest summary
Posted by le...@apache.org.
Fixing shifted fields and table schema for flow ingest summary
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/73b71619
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/73b71619
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/73b71619
Branch: refs/heads/SPOT-35_graphql_api
Commit: 73b716196d1b98dded849d7c540eb601fd9c0f87
Parents: ffa5b87
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Fri Jun 23 18:46:58 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/create_flow_migration_tables.hql | 5 +++--
spot-setup/migration/migrate_old_flow_data.sh | 2 +-
2 files changed, 4 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/73b71619/spot-setup/migration/create_flow_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_flow_migration_tables.hql b/spot-setup/migration/create_flow_migration_tables.hql
index 6e87ec3..453d14f 100644
--- a/spot-setup/migration/create_flow_migration_tables.hql
+++ b/spot-setup/migration/create_flow_migration_tables.hql
@@ -11,10 +11,11 @@ dstip STRING,
sport INT,
dport INT,
proto STRING,
-flag STRING,
ipkt INT,
ibyt INT,
-lda_score FLOAT,
+opkt INT,
+obyt INT,
+score FLOAT,
rank INT,
srcIpInternal INT,
destIpInternal INT,
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/73b71619/spot-setup/migration/migrate_old_flow_data.sh
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.sh b/spot-setup/migration/migrate_old_flow_data.sh
index 42534bb..4a7e04c 100755
--- a/spot-setup/migration/migrate_old_flow_data.sh
+++ b/spot-setup/migration/migrate_old_flow_data.sh
@@ -76,7 +76,7 @@ do
hive -e "$command"
command="INSERT INTO $DEST_DB.flow_scores PARTITION (y=$y, m=$m, d=$d)
-select tstart,srcip,dstip,sport,dport,proto,ipkt,ibyt,0,0,lda_score,rank,srcIpInternal,destIpInternal,srcGeo,dstGeo,srcDomain,dstDomain,srcIP_rep,dstIP_rep
+select tstart,srcip,dstip,sport,dport,proto,ipkt,ibyt,opkt,obyt,score,rank,srcIpInternal,destIpInternal,srcGeo,dstGeo,srcDomain,dstDomain,srcIP_rep,dstIP_rep
from $STAGING_DB.flow_scores_tmp;"
echo $command
hive -e "$command"
[15/20] incubator-spot git commit: Fixing missing hour and minute in
flow edge files migration
Posted by le...@apache.org.
Fixing missing hour and minute in flow edge files migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/57bf7364
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/57bf7364
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/57bf7364
Branch: refs/heads/SPOT-35_graphql_api
Commit: 57bf7364747059dded5c6342f6cfee626ed42dfa
Parents: 3b03df9
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Sat Jul 1 00:31:57 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/migrate_old_flow_data.py | 12 +++++++++---
1 file changed, 9 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/57bf7364/spot-setup/migration/migrate_old_flow_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.py b/spot-setup/migration/migrate_old_flow_data.py
index cc277ec..9d5ffc5 100755
--- a/spot-setup/migration/migrate_old_flow_data.py
+++ b/spot-setup/migration/migrate_old_flow_data.py
@@ -157,14 +157,20 @@ def main():
dest_table_name = 'flow_edge'
pattern = 'edge*.tsv'
edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
- filename = '{0}/{1}'.format(full_day_path, pattern)
- if len(edge_files) > 0:
+ for file in edge_files:
+
+ parts = (re.findall("edge-(\S+).tsv", file)[0]).split('-')
+ hh = int(parts[2])
+ mn = int(parts[3])
+
+ log.info("Processing File: {0} with HH: {1} and MN: {2}".format(file, hh, mn))
+ filename = '{0}/{1}'.format(full_day_path, file)
load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
util.execute_hive_cmd(load_cmd, log)
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart, srcip, dstip, sport, dport, proto, flags, tos, ibyt, ipkt, input, output, rip, obyt, opkt, 0, 0 FROM {5}.{6} WHERE srcip is not NULL;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart, srcip, dstip, sport, dport, proto, flags, tos, ibyt, ipkt, input, output, rip, obyt, opkt, {5}, {6} FROM {7}.{8} WHERE srcip is not NULL;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, hh, mn, staging_db, staging_table_name)
util.execute_hive_cmd(insert_cmd, log)
[04/20] incubator-spot git commit: Adding spot.conf migration and
fixing ipython templates migration
Posted by le...@apache.org.
Adding spot.conf migration and fixing ipython templates migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/40a587cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/40a587cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/40a587cd
Branch: refs/heads/SPOT-35_graphql_api
Commit: 40a587cde855ab2f6474b693a801ffcba8b77269
Parents: b751eba
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Thu Jun 29 18:41:20 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/README.md | 15 ++-
spot-setup/migrate_spot_0_9_to_1_0.py | 90 +++++++++++++
spot-setup/migration/.DS_Store | Bin 0 -> 6148 bytes
spot-setup/migration/README.md | 4 +-
spot-setup/migration/__init__.py | 1 +
spot-setup/migration/migrate_old_data.py | 83 ------------
spot-setup/migration/migrate_old_dns_data.py | 18 +--
spot-setup/migration/migrate_old_flow_data.py | 19 +--
spot-setup/migration/migrate_old_proxy_data.py | 17 ++-
spot-setup/migration/spot_conf_migration.py | 135 ++++++++++++++++++++
10 files changed, 268 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/README.md b/spot-setup/README.md
index 20edbda..b9b9147 100644
--- a/spot-setup/README.md
+++ b/spot-setup/README.md
@@ -15,6 +15,7 @@ Spot-setup contains the scripts to setup HDFS for Apache Spot solution. It will
To collaborate and run spot-setup, it is required the following prerequisites:
- A running Hadoop cluster
- Linux user account created in all nodes with sudo privileges
+- Enabling HDFS Access Control Lists
## General Description
@@ -26,7 +27,7 @@ The main script in the repository is **hdfs_setup.sh** which is responsible of l
This file also contains sources desired to be installed as part of Apache Spot, general paths for HDFS folders, and local paths in the Linux filesystem for the user as well as for machine learning, ipython, lda and ingest processes.
-To read more about these variables, please review the [documentation] (http://spot.incubator.apache.org/doc/#configuration).
+To read more about these variables, please review the [documentation](http://spot.incubator.apache.org/doc/#configuration).
## Database Query Scripts
@@ -71,6 +72,10 @@ To get to parquet format we need a staging table to store CSV data temporarily f
- proxy_timeline
+## Migrating OA Data from Spot 0.9 to 1.0
+
+Please review migration documentation [here](migrating/README.md).
+
## Licensing
spot-setup is licensed under Apache Version 2.0
@@ -81,11 +86,11 @@ Create a pull request and contact the maintainers.
## Issues
-- Create an [issue] (https://issues.apache.org/jira/browse/SPOT-20?jql=project%20%3D%20SPOT).
-- Go to our Slack [channel] (https://apachespot.slack.com/messages/general).
+- Create an [issue](https://issues.apache.org/jira/browse/SPOT-20?jql=project%20%3D%20SPOT).
+- Go to our Slack [channel](https://apachespot.slack.com/messages/general).
## Maintainers
-- [Moises Valdovinos] (https://github.com/moy8011)
-- [Everardo Lopez Sandoval] (https://github.com/EverLoSa)
+- [Moises Valdovinos](https://github.com/moy8011)
+- [Everardo Lopez Sandoval](https://github.com/EverLoSa)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migrate_spot_0_9_to_1_0.py
----------------------------------------------------------------------
diff --git a/spot-setup/migrate_spot_0_9_to_1_0.py b/spot-setup/migrate_spot_0_9_to_1_0.py
new file mode 100755
index 0000000..abb4eaf
--- /dev/null
+++ b/spot-setup/migrate_spot_0_9_to_1_0.py
@@ -0,0 +1,90 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+from migration.utilities import util
+
+pipelines=sys.argv[1]
+old_oa_path=sys.argv[2]
+staging_db=sys.argv[3]
+hdfs_staging_path=sys.argv[4]
+dest_db = sys.argv[5]
+impala_daemon = sys.argv[6]
+
+
+# Execution example:
+#./migrate_old_data.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(cur_path)[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Executing hdfs_setup.sh for initial setup")
+ util.execute_cmd('./hdfs_setup.sh',log)
+
+ log.info("Updating /etc/spot.conf with new changes in Spot 1.0")
+ util.execute_cmd("migration/spot_conf_migration.py '/etc/spot.conf' '{0}/spot.conf'".format(cur_path),log)
+
+ log.info("Copy ingest_conf.json file from old folder code to new Spot location")
+ util.execute_cmd('cp {0}/spot-ingest/ingest_conf.json {1}/spot-ingest/ingest_conf.json'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Copy Spot OA configuration files to new Spot location")
+ util.execute_cmd('cp {0}/spot-oa/oa/components/iana/iana_config.json {1}/spot-oa/oa/components/iana/iana_config.json'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/oa/components/reputation/reputation_config.json {1}/spot-oa/oa/components/reputation/reputation_config.json'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/oa/components/nc/nc_config.json {1}/spot-oa/oa/components/nc/nc_config.json'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/oa/components/data/engine.json {1}/spot-oa/oa/components/data/engine.json'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Copy Spot OA context files to new Spot location")
+ util.execute_cmd('cp {0}/spot-oa/context/iploc.csv {1}/spot-oa/context/iploc.csv'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/context/ipranges.csv {1}/spot-oa/context/ipranges.csv'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Install browserify and uglifyjs using npm")
+ os.chdir('{0}/ui'.format(new_oa_path))
+ util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
+
+ log.info("Install requirements using pip")
+ os.chdir('{0}'.format(new_oa_path))
+ util.execute_cmd('sudo pip install -r requirements.txt',log)
+
+
+ log.info("Migrate data from pipelines")
+ l_pipelines = pipelines.split(',')
+ valid_pipelines = ('flow', 'dns', 'proxy')
+ if len(l_pipelines) > 0:
+ for pipe in l_pipelines:
+ if pipe in valid_pipelines:
+ log.info("Migrating {0} old data to new spot release".format(pipe))
+ os.chdir('{0}/spot-setup/migration'.format(new_spot_path))
+ util.execute_cmd("python migrate_old_{0}_data.py '{1}' '{2}' '{3}' '{4}' '{5}' ".format(pipe, old_oa_path, staging_db, hdfs_staging_path, dest_db, impala_daemon),log)
+ else:
+ log.error("Pipeline {0} is not valid. Valid pipelines are flow, dns or proxy".format(pipe))
+ else:
+ log.error("Pipeline arguments must be separated by commas")
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/.DS_Store
----------------------------------------------------------------------
diff --git a/spot-setup/migration/.DS_Store b/spot-setup/migration/.DS_Store
new file mode 100644
index 0000000..5008ddf
Binary files /dev/null and b/spot-setup/migration/.DS_Store differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/migration/README.md b/spot-setup/migration/README.md
index f703697..8980c61 100644
--- a/spot-setup/migration/README.md
+++ b/spot-setup/migration/README.md
@@ -54,7 +54,7 @@ There is a launch and single script that will migrate all specified pipelines. T
### Execution
```python
-./migrate_old_data.py PIPELINES OLD_OA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
+./migrate_spot_0_9_to_1_0.py PIPELINES OLD_OA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
```
where variables mean:
@@ -67,5 +67,5 @@ where variables mean:
Example:
```python
-./migrate_old_data.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+./migrate_spot_0_9_to_1_0.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
```
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/__init__.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/__init__.py b/spot-setup/migration/__init__.py
new file mode 100644
index 0000000..8b13789
--- /dev/null
+++ b/spot-setup/migration/__init__.py
@@ -0,0 +1 @@
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/migrate_old_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_data.py b/spot-setup/migration/migrate_old_data.py
deleted file mode 100755
index c5b7b71..0000000
--- a/spot-setup/migration/migrate_old_data.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import sys
-import subprocess
-from utilities import util
-
-pipelines=sys.argv[1]
-old_oa_path=sys.argv[2]
-staging_db=sys.argv[3]
-hdfs_staging_path=sys.argv[4]
-dest_db = sys.argv[5]
-impala_daemon = sys.argv[6]
-
-
-# Execution example:
-#./migrate_old_data.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
-
-def main():
- log = util.get_logger('SPOT.MIGRATE')
-
- cur_path = os.path.dirname(os.path.realpath(__file__))
- new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
- new_oa_path = '{0}/spot-oa'.format(new_spot_path)
- log.info('New Spot OA path: {0}'.format(new_oa_path))
- old_spot_path = os.path.split(old_oa_path)[0]
-
-
- log.info("Copy ingest_conf.json file from old folder code to new Spot location")
- util.execute_cmd('cp {0}/spot-ingest/ingest_conf.json {1}/spot-ingest/ingest_conf.json'.format(old_spot_path, new_spot_path),log)
-
- log.info("Copy Spot OA configuration files to new Spot location")
- util.execute_cmd('cp {0}/spot-oa/oa/components/iana/iana_config.json {1}/spot-oa/oa/components/iana/iana_config.json'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/oa/components/reputation/reputation_config.json {1}/spot-oa/oa/components/reputation/reputation_config.json'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/oa/components/nc/nc_config.json {1}/spot-oa/oa/components/nc/nc_config.json'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/oa/components/data/engine.json {1}/spot-oa/oa/components/data/engine.json'.format(old_spot_path, new_spot_path),log)
-
- log.info("Copy Spot OA context files to new Spot location")
- util.execute_cmd('cp {0}/spot-oa/context/iploc.csv {1}/spot-oa/context/iploc.csv'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/context/ipranges.csv {1}/spot-oa/context/ipranges.csv'.format(old_spot_path, new_spot_path),log)
-
- log.info("Install browserify and uglifyjs using npm")
- os.chdir('{0}/ui'.format(new_oa_path))
- util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
-
- log.info("Install requirements using pip")
- os.chdir('{0}'.format(new_oa_path))
- util.execute_cmd('sudo pip install -r requirements.txt',log)
-
-
-
- l_pipelines = pipelines.split(',')
- valid_pipelines = ('flow', 'dns', 'proxy')
- if len(l_pipelines) > 0:
- for pipe in l_pipelines:
- if pipe in valid_pipelines:
- os.chdir('{0}/spot-setup/migration'.format(new_spot_path))
- util.execute_cmd("python migrate_old_{0}_data.py '{1}' '{2}' '{3}' '{4}' '{5}' ".format(pipe, old_oa_path, staging_db, hdfs_staging_path, dest_db, impala_daemon),log)
- else:
- log.error("Pipeline {0} is not valid. Valid pipelines are flow, dns or proxy".format(pipe))
- else:
- log.error("Pipelines argument must be separated by commas")
-
-
-if __name__=='__main__':
- main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/migrate_old_dns_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.py b/spot-setup/migration/migrate_old_dns_data.py
index 34b792a..f664cb1 100755
--- a/spot-setup/migration/migrate_old_dns_data.py
+++ b/spot-setup/migration/migrate_old_dns_data.py
@@ -216,14 +216,16 @@ def main():
util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
-
- log.info("Copying advanced mode notebooks to each existing day in new Spot location")
- for folder in os.listdir('{0}/ipynb/dns/'.format(new_oa_path)):
- util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb {1}/ipynb/dns/{2}/Advanced_Mode.ipynb'.format(old_oa_path, new_oa_path, folder),log)
-
- log.info("Copying threat investigation ipynb template to each existing day in new Spot location")
- for folder in os.listdir('{0}/ipynb/dns/'.format(new_oa_path)):
- util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb {1}/ipynb/dns/{2}/Threat_Investigation.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+ log.info("Creating ipynb template structure and copying advanced mode and threat investigation ipynb templates for each pre-existing day in the new Spot location")
+ ipynb_pipeline_path = '{0}/ipynb/dns/'.format(old_oa_path)
+ if os.path.exists(ipynb_pipeline_path):
+ for folder in os.listdir(ipynb_pipeline_path):
+ log.info("Creating ipynb dns folders in new Spot locaiton: {0}".format(folder))
+ util.execute_cmd('mkdir -p {0}/ipynb/dns/{1}/'.format(new_oa_path, folder),log)
+ log.info("Copying advanced mode ipynb template")
+ util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb {0}/ipynb/dns/{1}/Advanced_Mode.ipynb'.format(new_oa_path, folder),log)
+ log.info("Copying threat investigation ipynb template")
+ util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb {0}/ipynb/dns/{1}/Threat_Investigation.ipynb'.format(new_oa_path, folder),log)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/migrate_old_flow_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.py b/spot-setup/migration/migrate_old_flow_data.py
index eb9253f..cc277ec 100755
--- a/spot-setup/migration/migrate_old_flow_data.py
+++ b/spot-setup/migration/migrate_old_flow_data.py
@@ -216,15 +216,16 @@ def main():
util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
-
- log.info("Copying advanced mode notebooks to each existing day in new Spot location")
- for folder in os.listdir('{0}/ipynb/flow/'.format(new_oa_path)):
- util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb {1}/ipynb/flow/{2}/Advanced_Mode.ipynb'.format(old_oa_path, new_oa_path, folder),log)
-
- log.info("Copying threat investigation ipynb template to each existing day in new Spot location")
- for folder in os.listdir('{0}/ipynb/flow/'.format(new_oa_path)):
- util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb {1}/ipynb/flow/{2}/Threat_Investigation.ipynb'.format(old_oa_path, new_oa_path, folder),log)
-
+ log.info("Creating ipynb template structure and copying advanced mode and threat investigation ipynb templates for each pre-existing day in the new Spot location")
+ ipynb_pipeline_path = '{0}/ipynb/flow/'.format(old_oa_path)
+ if os.path.exists(ipynb_pipeline_path):
+ for folder in os.listdir(ipynb_pipeline_path):
+ log.info("Creating ipynb flow folders in new Spot locaiton: {0}".format(folder))
+ util.execute_cmd('mkdir -p {0}/ipynb/flow/{1}/'.format(new_oa_path, folder),log)
+ log.info("Copying advanced mode ipynb template")
+ util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb {0}/ipynb/flow/{1}/Advanced_Mode.ipynb'.format(new_oa_path, folder),log)
+ log.info("Copying threat investigation ipynb template")
+ util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb {0}/ipynb/flow/{1}/Threat_Investigation.ipynb'.format(new_oa_path, folder),log)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/migrate_old_proxy_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.py b/spot-setup/migration/migrate_old_proxy_data.py
index 4298fc6..ada7e6c 100755
--- a/spot-setup/migration/migrate_old_proxy_data.py
+++ b/spot-setup/migration/migrate_old_proxy_data.py
@@ -208,13 +208,16 @@ def main():
util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
- log.info("Copying advanced mode notebooks to each existing day in new Spot location")
- for folder in os.listdir('{0}/ipynb/proxy/'.format(new_oa_path)):
- util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb {1}/ipynb/proxy/{2}/Advanced_Mode.ipynb'.format(old_oa_path, new_oa_path, folder),log)
-
- log.info("Copying threat investigation ipynb template to each existing day in new Spot location")
- for folder in os.listdir('{0}/ipynb/proxy/'.format(new_oa_path)):
- util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb {1}/ipynb/proxy/{2}/Threat_Investigation.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+ log.info("Creating ipynb template structure and copying advanced mode and threat investigation ipynb templates for each pre-existing day in the new Spot location")
+ ipynb_pipeline_path = '{0}/ipynb/proxy/'.format(old_oa_path)
+ if os.path.exists(ipynb_pipeline_path):
+ for folder in os.listdir(ipynb_pipeline_path):
+ log.info("Creating ipynb proxy folders in new Spot locaiton: {0}".format(folder))
+ util.execute_cmd('mkdir -p {0}/ipynb/proxy/{1}/'.format(new_oa_path, folder),log)
+ log.info("Copying advanced mode ipynb template")
+ util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb {0}/ipynb/proxy/{1}/Advanced_Mode.ipynb'.format(new_oa_path, folder),log)
+ log.info("Copying threat investigation ipynb template")
+ util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb {0}/ipynb/proxy/{1}/Threat_Investigation.ipynb'.format(new_oa_path, folder),log)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/40a587cd/spot-setup/migration/spot_conf_migration.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/spot_conf_migration.py b/spot-setup/migration/spot_conf_migration.py
new file mode 100755
index 0000000..1ee927b
--- /dev/null
+++ b/spot-setup/migration/spot_conf_migration.py
@@ -0,0 +1,135 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import ConfigParser
+import sys
+import os
+from shutil import copyfile
+from utilities import util
+
+
+def main():
+
+ if len(sys.argv[1:]) < 2:
+ print "Please provide paths to: old_spot.conf , new_spot.conf"
+ sys.exit(1)
+
+ old_conf_file = sys.argv[1]
+ new_conf_file = sys.argv[2]
+
+ log = util.get_logger('SPOT.MIGRATE.CONF')
+
+ old_path = os.path.dirname(os.path.realpath(old_conf_file))
+
+ # create backup for the current configuration file.
+ log.info("Create a backup of /etc/spot.conf before changing it")
+ util.execute_cmd('sudo cp {0} {1}/spot.conf.bkp_0_9'.format(old_conf_file, old_path),log)
+
+ # create configuration objects.
+ old_config = ConfigParser.ConfigParser()
+ current_config = ConfigParser.ConfigParser()
+ new_config = ConfigParser.ConfigParser()
+
+ old_config.readfp(SecHead(open(old_conf_file)))
+ current_config.readfp(SecHead(open(new_conf_file)))
+
+ # create the new conf file.
+ new_config.add_section('conf')
+ for (k,v) in current_config.items("conf"):
+ if old_config.has_option('conf',k):
+ new_config.set('conf',k, old_config.get('conf',k))
+ else:
+ new_config.set('conf',k,v)
+
+ new_path = os.path.dirname(os.path.realpath(new_conf_file))
+ updated_conf_file = '{0}/spot.conf.new'.format(new_path)
+ log.info("Generating merged configuration file in {0}".format(updated_conf_file))
+ formatter(updated_conf_file,new_config)
+
+ log.info("Updating original spot.conf with new and migrated variables and values")
+ util.execute_cmd('sudo cp {0} {1}/spot.conf'.format(updated_conf_file, old_path),log)
+ util.execute_cmd('sudo chmod 0755 {0}/spot.conf'.format(old_path),log)
+
+
+def formatter(conf_file,conf):
+
+ with open(conf_file,'wb') as file:
+
+ file.write("# node configuration\n")
+ file.write("{0}={1}\n".format('UINODE',conf.get('conf','uinode')))
+ file.write("{0}={1}\n".format('MLNODE',conf.get('conf','MLNODE')))
+ file.write("{0}={1}\n".format('GWNODE',conf.get('conf','GWNODE')))
+ file.write("{0}={1}\n".format('DBNAME',conf.get('conf','DBNAME')))
+
+ file.write("# hdfs - base user and data source config\n")
+ file.write("{0}={1}\n".format('HUSER',conf.get('conf','HUSER')))
+ file.write("{0}={1}\n".format('NAME_NODE',conf.get('conf','NAME_NODE')))
+ file.write("{0}={1}\n".format('WEB_PORT',conf.get('conf','WEB_PORT')))
+ file.write("{0}={1}\n".format('DNS_PATH',conf.get('conf','DNS_PATH')))
+ file.write("{0}={1}\n".format('PROXY_PATH',conf.get('conf','PROXY_PATH')))
+ file.write("{0}={1}\n".format('FLOW_PATH',conf.get('conf','FLOW_PATH')))
+ file.write("{0}={1}\n".format('HPATH',conf.get('conf','HPATH')))
+
+ file.write("# impala config\n")
+ file.write("{0}={1}\n".format('IMPALA_DEM',conf.get('conf','IMPALA_DEM')))
+ file.write("{0}={1}\n".format('IMPALA_PORT',conf.get('conf','IMPALA_PORT')))
+
+ file.write("# local fs base user and data source config\n")
+ file.write("{0}={1}\n".format('LUSER',conf.get('conf','LUSER')))
+ file.write("{0}={1}\n".format('LPATH',conf.get('conf','LPATH')))
+ file.write("{0}={1}\n".format('RPATH',conf.get('conf','RPATH')))
+ file.write("{0}={1}\n".format('LIPATH',conf.get('conf','LIPATH')))
+
+ file.write("# suspicious connects config\n")
+ file.write("{0}={1}\n".format('USER_DOMAIN',conf.get('conf','USER_DOMAIN')))
+ file.write("{0}={1}\n".format('SPK_EXEC',conf.get('conf','SPK_EXEC')))
+ file.write("{0}={1}\n".format('SPK_EXEC_MEM',conf.get('conf','SPK_EXEC_MEM')))
+ file.write("{0}={1}\n".format('SPK_DRIVER_MEM',conf.get('conf','SPK_DRIVER_MEM')))
+ file.write("{0}={1}\n".format('SPK_DRIVER_MAX_RESULTS',conf.get('conf','SPK_DRIVER_MAX_RESULTS')))
+ file.write("{0}={1}\n".format('SPK_EXEC_CORES',conf.get('conf','SPK_EXEC_CORES')))
+ file.write("{0}={1}\n".format('SPK_DRIVER_MEM_OVERHEAD',conf.get('conf','SPK_DRIVER_MEM_OVERHEAD')))
+ file.write("{0}={1}\n".format('SPK_EXEC_MEM_OVERHEAD',conf.get('conf','SPK_EXEC_MEM_OVERHEAD')))
+ file.write("{0}={1}\n".format('SPK_AUTO_BRDCST_JOIN_THR',conf.get('conf','SPK_AUTO_BRDCST_JOIN_THR')))
+ file.write("{0}={1}\n".format('LDA_OPTIMIZER',conf.get('conf','LDA_OPTIMIZER')))
+ file.write("{0}={1}\n".format('LDA_ALPHA',conf.get('conf','LDA_ALPHA')))
+ file.write("{0}={1}\n".format('LDA_BETA',conf.get('conf','LDA_BETA')))
+ file.write("{0}={1}\n".format('PRECISION',conf.get('conf','PRECISION')))
+ file.write("{0}={1}\n".format('TOL',conf.get('conf','TOL')))
+ file.write("{0}={1}\n".format('TOPIC_COUNT',conf.get('conf','TOPIC_COUNT')))
+ file.write("{0}={1}\n".format('DUPFACTOR',conf.get('conf','DUPFACTOR')))
+
+class SecHead(object):
+
+ def __init__(self, fp):
+ self.fp = fp
+ self.sechead = '[conf]\n'
+
+ def readline(self):
+ if self.sechead:
+ try:
+ return self.sechead
+ finally:
+ self.sechead = None
+ else:
+ return self.fp.readline()
+
+if __name__ == '__main__':
+ main()
+
+
[02/20] incubator-spot git commit: Fixing typo in hdfs setup user
Posted by le...@apache.org.
Fixing typo in hdfs setup user
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/49be8bf8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/49be8bf8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/49be8bf8
Branch: refs/heads/SPOT-35_graphql_api
Commit: 49be8bf89222d1a62839379b96ed537f920df877
Parents: 65c58d3
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Tue Jul 4 00:52:37 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/hdfs_setup.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/49be8bf8/spot-setup/hdfs_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/hdfs_setup.sh b/spot-setup/hdfs_setup.sh
index 1ac0605..ac7b075 100755
--- a/spot-setup/hdfs_setup.sh
+++ b/spot-setup/hdfs_setup.sh
@@ -55,7 +55,7 @@ do
# Modifying permission on HDFS folders to allow Impala to read/write
hdfs dfs -chmod -R 775 ${HUSER}/$d
sudo -u hdfs hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
- sudo -u hdfs hdfs dfs -setfacl -R -m user:spotqa:rwx ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
done
# Creating Spot Database
[20/20] incubator-spot git commit: Fixing HDFS ACL permissions for
migration
Posted by le...@apache.org.
Fixing HDFS ACL permissions for migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/65c58d3b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/65c58d3b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/65c58d3b
Branch: refs/heads/SPOT-35_graphql_api
Commit: 65c58d3bcc7da827bc5d5d5310ab979b22e0a4e1
Parents: 9d0b091
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Tue Jul 4 00:45:08 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/hdfs_setup.sh | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/65c58d3b/spot-setup/hdfs_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/hdfs_setup.sh b/spot-setup/hdfs_setup.sh
index 4d303a5..1ac0605 100755
--- a/spot-setup/hdfs_setup.sh
+++ b/spot-setup/hdfs_setup.sh
@@ -54,7 +54,8 @@ do
# Modifying permission on HDFS folders to allow Impala to read/write
hdfs dfs -chmod -R 775 ${HUSER}/$d
- hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:spotqa:rwx ${HUSER}/$d
done
# Creating Spot Database
[18/20] incubator-spot git commit: Removing double quotes from DNS
Edge
Posted by le...@apache.org.
Removing double quotes from DNS Edge
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/9d0b0917
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/9d0b0917
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/9d0b0917
Branch: refs/heads/SPOT-35_graphql_api
Commit: 9d0b0917c694231039dc4018edd81e2acce6296b
Parents: a9b66c4
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Tue Jul 4 00:15:04 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/migrate_old_dns_data.py | 11 +++++++++--
1 file changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/9d0b0917/spot-setup/migration/migrate_old_dns_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.py b/spot-setup/migration/migrate_old_dns_data.py
index 15c8741..1c903de 100755
--- a/spot-setup/migration/migrate_old_dns_data.py
+++ b/spot-setup/migration/migrate_old_dns_data.py
@@ -162,14 +162,21 @@ def main():
mn = int(parts[-1])
log.info("Processing File: {0} with HH: {1} and MN: {2}".format(file, hh, mn))
- filename = '{0}/{1}'.format(full_day_path, file)
+
+ log.info("Removing double quotes File: {0}".format(file))
+ fixed_file = '{0}.fixed'.format(file)
+ sed_cmd = "sed 's/\"//g' {0}/{1} > {0}/{2}".format(full_day_path, file, fixed_file)
+ util.execute_cmd(sed_cmd, log)
+ filename = '{0}/{1}'.format(full_day_path, fixed_file)
load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
util.execute_hive_cmd(load_cmd, log)
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp(regexp_replace(frame_time, '\"', ''), 'MMMMM dd yyyy H:mm:ss.SSS z'), frame_len, ip_dst, ip_src, dns_qry_name, '', '0', '0', dns_a, {5}, dns_qry_class, dns_qry_type, dns_qry_rcode, '0' FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, hh, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp(frame_time, 'MMMMM dd yyyy H:mm:ss.SSS z'), frame_len, ip_dst, ip_src, dns_qry_name, '', '0', '0', dns_a, {5}, dns_qry_class, dns_qry_type, dns_qry_rcode, '0' FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, hh, staging_db, staging_table_name)
util.execute_hive_cmd(insert_cmd, log)
+ os.remove(filename)
+
##dns_storyboard
log.info("Processing Dns Storyboard")
[09/20] incubator-spot git commit: Delete .DS_Store
Posted by le...@apache.org.
Delete .DS_Store
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/d40daacb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/d40daacb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/d40daacb
Branch: refs/heads/SPOT-35_graphql_api
Commit: d40daacbc251d8b43a62456dde364706a1815eab
Parents: 40a587c
Author: Moy Valdovinos <mo...@intel.com>
Authored: Thu Jun 29 18:43:39 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/.DS_Store | Bin 6148 -> 0 bytes
1 file changed, 0 insertions(+), 0 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/d40daacb/spot-setup/migration/.DS_Store
----------------------------------------------------------------------
diff --git a/spot-setup/migration/.DS_Store b/spot-setup/migration/.DS_Store
deleted file mode 100644
index 5008ddf..0000000
Binary files a/spot-setup/migration/.DS_Store and /dev/null differ
[12/20] incubator-spot git commit: Fixing npm install
Posted by le...@apache.org.
Fixing npm install
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/475913d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/475913d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/475913d5
Branch: refs/heads/SPOT-35_graphql_api
Commit: 475913d5513c7b5ff534ce41c6a737b3b002bda1
Parents: f9352c5
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Fri Jun 30 12:49:55 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migrate_spot_0_9_to_1_0.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/475913d5/spot-setup/migrate_spot_0_9_to_1_0.py
----------------------------------------------------------------------
diff --git a/spot-setup/migrate_spot_0_9_to_1_0.py b/spot-setup/migrate_spot_0_9_to_1_0.py
index abb4eaf..a88d566 100755
--- a/spot-setup/migrate_spot_0_9_to_1_0.py
+++ b/spot-setup/migrate_spot_0_9_to_1_0.py
@@ -65,6 +65,7 @@ def main():
log.info("Install browserify and uglifyjs using npm")
os.chdir('{0}/ui'.format(new_oa_path))
util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
+ util.execute_cmd('npm install',log)
log.info("Install requirements using pip")
os.chdir('{0}'.format(new_oa_path))
[11/20] incubator-spot git commit: Fixing link in documentation
Posted by le...@apache.org.
Fixing link in documentation
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/f9352c59
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/f9352c59
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/f9352c59
Branch: refs/heads/SPOT-35_graphql_api
Commit: f9352c592656b58da47dadcf20d7284e217675ca
Parents: d40daac
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Thu Jun 29 18:48:51 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/f9352c59/spot-setup/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/README.md b/spot-setup/README.md
index b9b9147..2b803ab 100644
--- a/spot-setup/README.md
+++ b/spot-setup/README.md
@@ -74,7 +74,7 @@ To get to parquet format we need a staging table to store CSV data temporarily f
## Migrating OA Data from Spot 0.9 to 1.0
-Please review migration documentation [here](migrating/README.md).
+Please review migration documentation [here](migration/README.md).
## Licensing
[16/20] incubator-spot git commit: Adding resp codes to Proxy Edge
with lookup on IANA resp code table
Posted by le...@apache.org.
Adding resp codes to Proxy Edge with lookup on IANA resp code table
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/a9b66c44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/a9b66c44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/a9b66c44
Branch: refs/heads/SPOT-35_graphql_api
Commit: a9b66c44efda4f4e76c43b3c7034bc25ffc329d8
Parents: bbab834
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Mon Jul 3 21:42:00 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/migrate_old_proxy_data.py | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/a9b66c44/spot-setup/migration/migrate_old_proxy_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.py b/spot-setup/migration/migrate_old_proxy_data.py
index 06ef5f2..61f2e98 100755
--- a/spot-setup/migration/migrate_old_proxy_data.py
+++ b/spot-setup/migration/migrate_old_proxy_data.py
@@ -131,6 +131,14 @@ def main():
util.execute_hive_cmd(insert_cmd, log)
+ ## proxy IANA resp codes
+ log.info("Uploading Proxy IANA resp codes")
+ iana_table_name = 'proxy_iana_rcode_tmp'
+ iana_code_csv = '{0}/oa/components/iana/http-rcode.csv'.format(old_oa_path)
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(iana_code_csv, staging_db, iana_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+
## proxy Edge
log.info("Processing Proxy Edge")
staging_table_name = 'proxy_edge_tmp'
@@ -144,7 +152,7 @@ def main():
load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
util.execute_hive_cmd(load_cmd, log)
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, time, clientip, host, webcat, '0', reqmethod, useragent, resconttype, referer, uriport, serverip, scbytes, csbytes, fulluri, hour(time), respcode FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT e.tdate, e.time, e.clientip, e.host, e.webcat, COALESCE(i.respcode,'0'), e.reqmethod, e.useragent, e.resconttype, e.referer, e.uriport, e.serverip, e.scbytes, e.csbytes, e.fulluri, hour(e.time), e.respcode FROM {5}.{6} e left join {5}.{7} i on e.respcode = i.respcode_name ;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name, iana_table_name)
util.execute_hive_cmd(insert_cmd, log)
@@ -165,13 +173,6 @@ def main():
##proxy Timeline
log.info("Processing Proxy Timeline")
-
- log.info("Uploading Proxy IANA resp codes")
- iana_table_name = 'proxy_iana_rcode_tmp'
- iana_code_csv = '{0}/oa/components/iana/http-rcode.csv'.format(old_oa_path)
- load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(iana_code_csv, staging_db, iana_table_name)
- util.execute_hive_cmd(load_cmd, log)
-
staging_table_name = 'proxy_timeline_tmp'
dest_table_name = 'proxy_timeline'
pattern = 'timeline*.tsv'
@@ -195,7 +196,7 @@ def main():
util.execute_hive_cmd(load_cmd, log)
# Insert into new table from staging table and adding FullURI value
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', t.tstart, t.tend, t.duration, t.clientip, t.respcode, i.respcode_name FROM {6}.{7} t left join {6}.{8} i on t.respcode = i.respcode where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name, iana_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', t.tstart, t.tend, t.duration, t.clientip, t.respcode, COALESCE(i.respcode_name,'') FROM {6}.{7} t left join {6}.{8} i on t.respcode = i.respcode where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name, iana_table_name)
util.execute_hive_cmd(insert_cmd, log)
else:
[10/20] incubator-spot git commit: Updating documentation for new
changes
Posted by le...@apache.org.
Updating documentation for new changes
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/b751eba2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/b751eba2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/b751eba2
Branch: refs/heads/SPOT-35_graphql_api
Commit: b751eba25a1aa74d383a1218e87f55e0a65d5c60
Parents: 1693f88
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Thu Jun 29 05:10:12 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/README.md | 15 ++++++++-------
spot-setup/migration/migrate_old_data.py | 2 +-
2 files changed, 9 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b751eba2/spot-setup/migration/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/migration/README.md b/spot-setup/migration/README.md
index d8d711d..f703697 100644
--- a/spot-setup/migration/README.md
+++ b/spot-setup/migration/README.md
@@ -1,4 +1,4 @@
-## Data Migration from Spot 0.9 to Spot 1.0
+## OA Data Migration from Spot 0.9 to Spot 1.0
This document is intended for any developer or sysadmin who wants to migrate their existing OA data from Spot 0.9 to Spot 1.0. In Spot 0.9, OA data was stored in CSV files in a given location in the server used for OA (specified in spot.conf during original installation). In Spot 1.0, OA data is stored in Impala tables. The purposes of these scripts are to migrate independently each use case (flow, proxy and dns) from those CSV files into the new Impala tables.
@@ -8,6 +8,7 @@ This migration process is optional and only for those users who want to keep usa
- You must run first new Spot setup installation to have the new tables created in Impala.
- You must log in to the OA server and run these scripts from there.
+- You must run these scripts from the spot-setup/migration folder in the new Spot 1.0 location
### CSV to Impala Tables Mapping
@@ -48,23 +49,23 @@ timeline-*.csv | proxy_timeline
### Data Flow
-These scripts will read each of the CSV from the existing location and import data to Impala tables accordingly, creating first a staging database and tables to load the records in the CSV and then insert that data into the new Spot 1.0 tables. You must execute this migration process from the server where Spot 0.9 CSV files are located. You may run one or all script for each use case (flow, dns and proxy) according to your needs and your existing data. At the end of each script, the specific pipeline folder will be moved from the original location to a backup folder. Staging tables and their respective HDFS paths will be removed.
+There is a launch and single script that will migrate all specified pipelines. This process will read each of the CSV from the existing location and import data to Impala tables accordingly, creating first a staging database and tables to load the records in the CSV and then insert that data into the new Spot 1.0 tables. You must execute this migration process from the server where Spot 0.9 CSV files are located. You may provide one pipeline or all (flow, dns and proxy) according to your needs and your existing data. At the end of each script, the old data pipeline folder will be moved from the original location to a backup folder. Staging tables and their respective HDFS paths will be removed.
### Execution
-```shell
-./migrate_old_dns_data.sh OLD_OA_PATH NEW_OA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
+```python
+./migrate_old_data.py PIPELINES OLD_OA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
```
where variables mean:
+- **PIPELINES** - Comma-separated list of the pipelines to be migrated
- **OLD_OA_PATH** - Path to the old Spot-OA (0.9 release) directory in the local filesystem
-- **NEW_OA_PATH** - Path to the new Spot-OA (1.0 release) directory in the local filesystem
- **STAGING_DB_NAME** - Name of the staging database to be created to temporarily store these records
- **STAGING_DB_HDFS_PATH** - HDFS path of the staging database to be created to temporarily store these records
- **NEW_SPOT_IMPALA_DB** - Database name of the Spot 1.0 Impala tables. Use the same as in the spot.conf when the new Spot release was installed
- **IMPALA_DAEMON** - Choose an Impala daemon to be used to run scripts' queries.
Example:
-```shell
-./migrate_old_flow_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+```python
+./migrate_old_data.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
```
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b751eba2/spot-setup/migration/migrate_old_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_data.py b/spot-setup/migration/migrate_old_data.py
index d37840b..c5b7b71 100755
--- a/spot-setup/migration/migrate_old_data.py
+++ b/spot-setup/migration/migrate_old_data.py
@@ -31,7 +31,7 @@ impala_daemon = sys.argv[6]
# Execution example:
-#./migrate_old_proxy_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+#./migrate_old_data.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
def main():
log = util.get_logger('SPOT.MIGRATE')
[13/20] incubator-spot git commit: Fixing pip requirements by adding
python-devel tools
Posted by le...@apache.org.
Fixing pip requirements by adding python-devel tools
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/435f1279
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/435f1279
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/435f1279
Branch: refs/heads/SPOT-35_graphql_api
Commit: 435f12794e420995e49a29d95ab39ec77cb2656a
Parents: 475913d
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Fri Jun 30 13:50:42 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migrate_spot_0_9_to_1_0.py | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/435f1279/spot-setup/migrate_spot_0_9_to_1_0.py
----------------------------------------------------------------------
diff --git a/spot-setup/migrate_spot_0_9_to_1_0.py b/spot-setup/migrate_spot_0_9_to_1_0.py
index a88d566..32ef3a4 100755
--- a/spot-setup/migrate_spot_0_9_to_1_0.py
+++ b/spot-setup/migrate_spot_0_9_to_1_0.py
@@ -67,9 +67,12 @@ def main():
util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
util.execute_cmd('npm install',log)
+ log.info("Install python-devel using yum")
+ util.execute_cmd('sudo yum install python-devel -y',log)
+
log.info("Install requirements using pip")
os.chdir('{0}'.format(new_oa_path))
- util.execute_cmd('sudo pip install -r requirements.txt',log)
+ util.execute_cmd('sudo -H -E pip install -r requirements.txt',log)
log.info("Migrate data from pipelines")
[17/20] incubator-spot git commit: Adding OA network context migration
Posted by le...@apache.org.
Adding OA network context migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/1fc93dec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/1fc93dec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/1fc93dec
Branch: refs/heads/SPOT-35_graphql_api
Commit: 1fc93deccfbd935e8f03956e9be6eaec7f572a78
Parents: 57bf736
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Mon Jul 3 15:28:39 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migrate_spot_0_9_to_1_0.py | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/1fc93dec/spot-setup/migrate_spot_0_9_to_1_0.py
----------------------------------------------------------------------
diff --git a/spot-setup/migrate_spot_0_9_to_1_0.py b/spot-setup/migrate_spot_0_9_to_1_0.py
index 32ef3a4..c6adef2 100755
--- a/spot-setup/migrate_spot_0_9_to_1_0.py
+++ b/spot-setup/migrate_spot_0_9_to_1_0.py
@@ -61,6 +61,7 @@ def main():
log.info("Copy Spot OA context files to new Spot location")
util.execute_cmd('cp {0}/spot-oa/context/iploc.csv {1}/spot-oa/context/iploc.csv'.format(old_spot_path, new_spot_path),log)
util.execute_cmd('cp {0}/spot-oa/context/ipranges.csv {1}/spot-oa/context/ipranges.csv'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/context/networkcontext_1.csv {1}/spot-oa/context/networkcontext_1.csv'.format(old_spot_path, new_spot_path),log)
log.info("Install browserify and uglifyjs using npm")
os.chdir('{0}/ui'.format(new_oa_path))
[14/20] incubator-spot git commit: Removing quotes from impala daemon
variable
Posted by le...@apache.org.
Removing quotes from impala daemon variable
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/3b03df9c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/3b03df9c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/3b03df9c
Branch: refs/heads/SPOT-35_graphql_api
Commit: 3b03df9c8e9e096b665c8f2baff17a9db10a69b2
Parents: 435f127
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Fri Jun 30 16:06:33 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/spot.conf | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/3b03df9c/spot-setup/spot.conf
----------------------------------------------------------------------
diff --git a/spot-setup/spot.conf b/spot-setup/spot.conf
index 4e706b3..5285eda 100755
--- a/spot-setup/spot.conf
+++ b/spot-setup/spot.conf
@@ -14,7 +14,7 @@ FLOW_PATH=${HUSER}/${DSOURCE}/hive/y=${YR}/m=${MH}/d=${DY}/
HPATH=${HUSER}/${DSOURCE}/scored_results/${FDATE}
#impala config
-IMPALA_DEM='node04'
+IMPALA_DEM=node04
IMPALA_PORT=21050
#local fs base user and data source config
[19/20] incubator-spot git commit: Fixing DNS Storyboard load and
data backing up
Posted by le...@apache.org.
Fixing DNS Storyboard load and data backing up
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/020d98cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/020d98cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/020d98cd
Branch: refs/heads/SPOT-35_graphql_api
Commit: 020d98cdde8a77a1133ca0d9490aaf396202e4e1
Parents: 49be8bf
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Tue Jul 4 14:44:10 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/create_dns_migration_tables.hql | 2 +-
spot-setup/migration/migrate_old_dns_data.py | 4 +++-
spot-setup/migration/migrate_old_flow_data.py | 3 ++-
spot-setup/migration/migrate_old_proxy_data.py | 3 ++-
4 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/020d98cd/spot-setup/migration/create_dns_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_dns_migration_tables.hql b/spot-setup/migration/create_dns_migration_tables.hql
index c85fd0f..b564486 100644
--- a/spot-setup/migration/create_dns_migration_tables.hql
+++ b/spot-setup/migration/create_dns_migration_tables.hql
@@ -77,7 +77,7 @@ dns_threat STRING,
title STRING,
text STRING
)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
LOCATION '${var:hpath}/dns/storyboard'
TBLPROPERTIES ('skip.header.line.count'='1');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/020d98cd/spot-setup/migration/migrate_old_dns_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.py b/spot-setup/migration/migrate_old_dns_data.py
index 1c903de..912f79b 100755
--- a/spot-setup/migration/migrate_old_dns_data.py
+++ b/spot-setup/migration/migrate_old_dns_data.py
@@ -223,7 +223,9 @@ def main():
log.info("Moving CSV data to backup folder")
util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
- util.execute_cmd('mv {0}/data/dns/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('cp -r {0}/data/dns/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('rm -r {0}/data/dns/'.format(old_oa_path),log)
+
log.info("Invalidating metadata in Impala to refresh tables content")
util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/020d98cd/spot-setup/migration/migrate_old_flow_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.py b/spot-setup/migration/migrate_old_flow_data.py
index 9d5ffc5..3b5543b 100755
--- a/spot-setup/migration/migrate_old_flow_data.py
+++ b/spot-setup/migration/migrate_old_flow_data.py
@@ -216,7 +216,8 @@ def main():
log.info("Moving CSV data to backup folder")
util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
- util.execute_cmd('mv {0}/data/flow/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('cp -r {0}/data/flow/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('rm -r {0}/data/flow/'.format(old_oa_path),log)
log.info("Invalidating metadata in Impala to refresh tables content")
util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/020d98cd/spot-setup/migration/migrate_old_proxy_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.py b/spot-setup/migration/migrate_old_proxy_data.py
index 61f2e98..51ef7b2 100755
--- a/spot-setup/migration/migrate_old_proxy_data.py
+++ b/spot-setup/migration/migrate_old_proxy_data.py
@@ -212,7 +212,8 @@ def main():
log.info("Moving CSV data to backup folder")
util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
- util.execute_cmd('mv {0}/data/proxy/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('cp -r {0}/data/proxy/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('rm -r {0}/data/proxy/'.format(old_oa_path),log)
log.info("Invalidating metadata in Impala to refresh tables content")
util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
[06/20] incubator-spot git commit: Adding migration of advanced mode
notebooks
Posted by le...@apache.org.
Adding migration of advanced mode notebooks
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/b829177c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/b829177c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/b829177c
Branch: refs/heads/SPOT-35_graphql_api
Commit: b829177c0dded007c286ad393124bbf59c668ae4
Parents: 73b7161
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Fri Jun 23 23:34:39 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/README.md | 16 +++++++-----
spot-setup/migration/migrate_old_dns_data.sh | 29 +++++++++++++--------
spot-setup/migration/migrate_old_flow_data.sh | 27 ++++++++++++-------
spot-setup/migration/migrate_old_proxy_data.sh | 27 ++++++++++++-------
4 files changed, 61 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b829177c/spot-setup/migration/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/migration/README.md b/spot-setup/migration/README.md
index d344322..d8d711d 100644
--- a/spot-setup/migration/README.md
+++ b/spot-setup/migration/README.md
@@ -1,12 +1,13 @@
## Data Migration from Spot 0.9 to Spot 1.0
-This document is intended for any developer or sysadmin who wants to migrate their existing data from Spot 0.9 to Spot 1.0. In Spot 0.9, data was stored in CSV files in a given location. In Spot 1.0, data is stored in Impala tables. The purposes of these scripts are to migrate independently each use case (flow, proxy and dns) from those CSV files the new Impala tables.
+This document is intended for any developer or sysadmin who wants to migrate their existing OA data from Spot 0.9 to Spot 1.0. In Spot 0.9, OA data was stored in CSV files in a given location in the server used for OA (specified in spot.conf during original installation). In Spot 1.0, OA data is stored in Impala tables. The purposes of these scripts are to migrate independently each use case (flow, proxy and dns) from those CSV files into the new Impala tables.
-This migration process is optional and only for those users who want to keep usable their data generated by ML and OA (scores, edges, chords, dendros, ingest summaries, storyboards, threat investigations and timelines) in the previous Spot version.
+This migration process is optional and only for those users who want to keep usable their OA data (scores, edges, chords, dendros, ingest summaries, storyboards, threat investigations and timelines) generated in the previous Spot version.
### Requirements
-You must run first new Spot setup installation to have the new tables created in Impala.
+- You must run first new Spot setup installation to have the new tables created in Impala.
+- You must log in to the OA server and run these scripts from there.
### CSV to Impala Tables Mapping
@@ -47,16 +48,17 @@ timeline-*.csv | proxy_timeline
### Data Flow
-These scripts will read each of the CSV from the existing location and import data to Impala tables accordingly, creating first a staging database and tables to load the records in the CSV and then insert that data into the new Spot 1.0 tables. You must execute this migration process from the server where Spot 0.9 CSV files are located. You may run one or all script for each use case (flow, dns and proxy) according to your needs and your existing data.
+These scripts will read each of the CSV from the existing location and import data to Impala tables accordingly, creating first a staging database and tables to load the records in the CSV and then insert that data into the new Spot 1.0 tables. You must execute this migration process from the server where Spot 0.9 CSV files are located. You may run one or all script for each use case (flow, dns and proxy) according to your needs and your existing data. At the end of each script, the specific pipeline folder will be moved from the original location to a backup folder. Staging tables and their respective HDFS paths will be removed.
### Execution
```shell
-./migrate_old_dns_data.sh OLD_DATA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
+./migrate_old_dns_data.sh OLD_OA_PATH NEW_OA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
```
where variables mean:
-- **OLD_DATA_PATH** - Path to existing Spot 0.9 CSV files in the local filesystem
+- **OLD_OA_PATH** - Path to the old Spot-OA (0.9 release) directory in the local filesystem
+- **NEW_OA_PATH** - Path to the new Spot-OA (1.0 release) directory in the local filesystem
- **STAGING_DB_NAME** - Name of the staging database to be created to temporarily store these records
- **STAGING_DB_HDFS_PATH** - HDFS path of the staging database to be created to temporarily store these records
- **NEW_SPOT_IMPALA_DB** - Database name of the Spot 1.0 Impala tables. Use the same as in the spot.conf when the new Spot release was installed
@@ -64,5 +66,5 @@ where variables mean:
Example:
```shell
-./migrate_old_dns_data.sh '/home/spot/spot-csv-data' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+./migrate_old_flow_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
```
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b829177c/spot-setup/migration/migrate_old_dns_data.sh
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.sh b/spot-setup/migration/migrate_old_dns_data.sh
index f517cbb..96341e8 100755
--- a/spot-setup/migration/migrate_old_dns_data.sh
+++ b/spot-setup/migration/migrate_old_dns_data.sh
@@ -17,14 +17,15 @@
# limitations under the License.
#
-OLD_DATA_PATH=$1
-STAGING_DB=$2
-HDFS_STAGING_PATH=$3
-DEST_DB=$4
-IMPALA_DEM=$5
+OLD_OA_PATH=$1
+NEW_OA_PATH=$2
+STAGING_DB=$3
+HDFS_STAGING_PATH=$4
+DEST_DB=$5
+IMPALA_DEM=$6
# Execution example:
-#./migrate_old_dns_data.sh '/home/spot/spot-csv-data' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+#./migrate_old_dns_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
hadoop fs -mkdir $HDFS_STAGING_PATH
hadoop fs -mkdir $HDFS_STAGING_PATH/dns/
@@ -44,7 +45,7 @@ impala-shell -i ${IMPALA_DEM} --var=hpath=${HDFS_STAGING_PATH} --var=dbname=${ST
## DNS Ingest Summary
echo "Processing DNS Ingest Summary"
-ing_sum_path=$OLD_DATA_PATH/dns/ingest_summary/is_??????.csv
+ing_sum_path=$OLD_OA_PATH/data/dns/ingest_summary/is_??????.csv
for file in $ing_sum_path
do
@@ -53,7 +54,7 @@ do
done
-DAYS=$OLD_DATA_PATH/dns/2*
+DAYS=$OLD_OA_PATH/data/dns/2*
for dir in $DAYS
do
@@ -153,7 +154,7 @@ select ip_threat, dns_threat, title, text from $STAGING_DB.dns_storyboard_tmp;"
for file in $threat_dendro_files
do
filename="$(basename $file)"
- ip="${filename%.tsv}"
+ ip="${filename%.csv}"
ip="${ip#threat-dendro-}"
echo $filename $ip
@@ -172,6 +173,12 @@ from $STAGING_DB.dns_threat_dendro_tmp;"
done
+## Copying advanced mode notebooks to new paths in Spot 1.0
+for new_dir in $NEW_OA_PATH/ipynb/dns/*
+do
+ cp $OLD_OA_PATH/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb $new_dir/Advanced_Mode.ipynb
+done
+
# Dropping staging tables
impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_dns_migration_tables.hql
@@ -179,8 +186,8 @@ impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_dns_migratio
hadoop fs -rm -r $HDFS_STAGING_PATH/dns/
# Moving CSV data to backup folder
-mkdir $OLD_DATA_PATH/backup/
-mv $OLD_DATA_PATH/dns/ $OLD_DATA_PATH/backup/
+mkdir $OLD_OA_PATH/data/backup/
+mv $OLD_OA_PATH/data/dns/ $OLD_OA_PATH/data/backup/
# Invalidating metadata in Impala to refresh tables content
impala-shell -i ${IMPALA_DEM} -q "INVALIDATE METADATA;"
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b829177c/spot-setup/migration/migrate_old_flow_data.sh
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.sh b/spot-setup/migration/migrate_old_flow_data.sh
index 4a7e04c..9b51987 100755
--- a/spot-setup/migration/migrate_old_flow_data.sh
+++ b/spot-setup/migration/migrate_old_flow_data.sh
@@ -17,14 +17,15 @@
# limitations under the License.
#
-OLD_DATA_PATH=$1
-STAGING_DB=$2
-HDFS_STAGING_PATH=$3
-DEST_DB=$4
-IMPALA_DEM=$5
+OLD_OA_PATH=$1
+NEW_OA_PATH=$2
+STAGING_DB=$3
+HDFS_STAGING_PATH=$4
+DEST_DB=$5
+IMPALA_DEM=$6
# Execution example:
-#./migrate_old_flow_data.sh '/home/spot/spot-csv-data' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+#./migrate_old_flow_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
hadoop fs -mkdir $HDFS_STAGING_PATH
hadoop fs -mkdir $HDFS_STAGING_PATH/flow/
@@ -45,7 +46,7 @@ impala-shell -i ${IMPALA_DEM} --var=hpath=${HDFS_STAGING_PATH} --var=dbname=${ST
## Flow Ingest Summary
echo "Processing Flow Ingest Summary"
-ing_sum_path=$OLD_DATA_PATH/flow/ingest_summary/is_??????.csv
+ing_sum_path=$OLD_OA_PATH/data/flow/ingest_summary/is_??????.csv
for file in $ing_sum_path
do
@@ -54,7 +55,7 @@ do
done
-DAYS=$OLD_DATA_PATH/flow/2*
+DAYS=$OLD_OA_PATH/data/flow/2*
for dir in $DAYS
do
@@ -180,6 +181,12 @@ select '$ip', tstart, tend, srcip, dstip, proto, sport, dport, ipkt, ibyt from $
done
+## Copying advanced mode notebooks to new paths in Spot 1.0
+for new_dir in $NEW_OA_PATH/ipynb/flow/*
+do
+ cp $OLD_OA_PATH/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb $new_dir/Advanced_Mode.ipynb
+done
+
# Dropping staging tables
impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_flow_migration_tables.hql
@@ -187,8 +194,8 @@ impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_flow_migrati
hadoop fs -rm -r $HDFS_STAGING_PATH/flow/
# Moving CSV data to backup folder
-mkdir $OLD_DATA_PATH/backup/
-mv $OLD_DATA_PATH/flow/ $OLD_DATA_PATH/backup/
+mkdir $OLD_OA_PATH/data/backup/
+mv $OLD_OA_PATH/data/flow/ $OLD_OA_PATH/data/backup/
# Invalidating metadata in Impala to refresh tables content
impala-shell -i ${IMPALA_DEM} -q "INVALIDATE METADATA;"
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/b829177c/spot-setup/migration/migrate_old_proxy_data.sh
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.sh b/spot-setup/migration/migrate_old_proxy_data.sh
index 0a93845..f8eaea3 100755
--- a/spot-setup/migration/migrate_old_proxy_data.sh
+++ b/spot-setup/migration/migrate_old_proxy_data.sh
@@ -17,14 +17,15 @@
# limitations under the License.
#
-OLD_DATA_PATH=$1
-STAGING_DB=$2
-HDFS_STAGING_PATH=$3
-DEST_DB=$4
-IMPALA_DEM=$5
+OLD_OA_PATH=$1
+NEW_OA_PATH=$2
+STAGING_DB=$3
+HDFS_STAGING_PATH=$4
+DEST_DB=$5
+IMPALA_DEM=$6
# Execution example:
-#./migrate_old_proxy_data.sh '/home/spot/spot-csv-data' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+#./migrate_old_proxy_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
hadoop fs -mkdir $HDFS_STAGING_PATH
hadoop fs -mkdir $HDFS_STAGING_PATH/proxy/
@@ -43,7 +44,7 @@ impala-shell -i ${IMPALA_DEM} --var=hpath=${HDFS_STAGING_PATH} --var=dbname=${ST
# Proxy Ingest Summary
echo "Processing proxy Ingest Summary"
-ing_sum_path=$OLD_DATA_PATH/proxy/ingest_summary/is_??????.csv
+ing_sum_path=$OLD_OA_PATH/data/proxy/ingest_summary/is_??????.csv
for file in $ing_sum_path
do
@@ -52,7 +53,7 @@ do
done
-DAYS=$OLD_DATA_PATH/proxy/2*
+DAYS=$OLD_OA_PATH/data/proxy/2*
for dir in $DAYS
do
@@ -146,6 +147,12 @@ from $STAGING_DB.proxy_storyboard_tmp;"
done
+## Copying advanced mode notebooks to new paths in Spot 1.0
+for new_dir in $NEW_OA_PATH/ipynb/proxy/*
+do
+ cp $OLD_OA_PATH/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb $new_dir/Advanced_Mode.ipynb
+done
+
# Dropping staging tables
impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_proxy_migration_tables.hql
@@ -153,8 +160,8 @@ impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_proxy_migrat
hadoop fs -rm -r $HDFS_STAGING_PATH/proxy/
# Moving CSV data to backup folder
-mkdir $OLD_DATA_PATH/backup/
-mv $OLD_DATA_PATH/proxy/ $OLD_DATA_PATH/backup/
+mkdir $OLD_OA_PATH/data/backup/
+mv $OLD_OA_PATH/data/proxy/ $OLD_OA_PATH/data/backup/
# Invalidating metadata in Impala to refresh tables content
impala-shell -i ${IMPALA_DEM} -q "INVALIDATE METADATA;"
[03/20] incubator-spot git commit: Fixing Proxy IANA resp codes and
DNS Edge hour
Posted by le...@apache.org.
Fixing Proxy IANA resp codes and DNS Edge hour
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/bbab8341
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/bbab8341
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/bbab8341
Branch: refs/heads/SPOT-35_graphql_api
Commit: bbab8341c706b211c13bbbc3dc608cba3766bd2c
Parents: 1fc93de
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Mon Jul 3 21:08:45 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/hdfs_setup.sh | 8 ++---
spot-setup/migrate_spot_0_9_to_1_0.py | 33 ++++++++++++--------
.../migration/create_proxy_migration_tables.hql | 11 +++++++
.../migration/drop_proxy_migration_tables.hql | 1 +
spot-setup/migration/migrate_old_dns_data.py | 12 +++++--
spot-setup/migration/migrate_old_proxy_data.py | 21 +++++++++----
6 files changed, 60 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bbab8341/spot-setup/hdfs_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/hdfs_setup.sh b/spot-setup/hdfs_setup.sh
index 9c557ba..4d303a5 100755
--- a/spot-setup/hdfs_setup.sh
+++ b/spot-setup/hdfs_setup.sh
@@ -51,11 +51,11 @@ do
echo "creating $d/$f"
hdfs dfs -mkdir ${HUSER}/$d/$f
done
-done
-# Modifying permission on HDFS folders to allow Impala to read/write
-hdfs dfs -chmod -R 775 ${HUSER}
-hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}
+ # Modifying permission on HDFS folders to allow Impala to read/write
+ hdfs dfs -chmod -R 775 ${HUSER}/$d
+ hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
+done
# Creating Spot Database
impala-shell -i ${IMPALA_DEM} -q "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bbab8341/spot-setup/migrate_spot_0_9_to_1_0.py
----------------------------------------------------------------------
diff --git a/spot-setup/migrate_spot_0_9_to_1_0.py b/spot-setup/migrate_spot_0_9_to_1_0.py
index c6adef2..1c00a4f 100755
--- a/spot-setup/migrate_spot_0_9_to_1_0.py
+++ b/spot-setup/migrate_spot_0_9_to_1_0.py
@@ -22,10 +22,10 @@ import sys
import subprocess
from migration.utilities import util
-pipelines=sys.argv[1]
-old_oa_path=sys.argv[2]
-staging_db=sys.argv[3]
-hdfs_staging_path=sys.argv[4]
+pipelines = sys.argv[1]
+old_oa_path = sys.argv[2]
+staging_db = sys.argv[3]
+hdfs_staging_path = sys.argv[4]
dest_db = sys.argv[5]
impala_daemon = sys.argv[6]
@@ -37,10 +37,17 @@ def main():
log = util.get_logger('SPOT.MIGRATE')
cur_path = os.path.dirname(os.path.realpath(__file__))
+
+ global old_oa_path
+ old_oa_path = old_oa_path.rstrip('/')
+ old_spot_path = os.path.split(old_oa_path)[0]
new_spot_path = os.path.split(cur_path)[0]
new_oa_path = '{0}/spot-oa'.format(new_spot_path)
- log.info('New Spot OA path: {0}'.format(new_oa_path))
- old_spot_path = os.path.split(old_oa_path)[0]
+
+ log.info('Old Spot path: {0}'.format(old_spot_path))
+ log.info('Old OA path: {0}'.format(old_oa_path))
+ log.info('New Spot path: {0}'.format(new_spot_path))
+ log.info('New OA path: {0}'.format(new_oa_path))
log.info("Executing hdfs_setup.sh for initial setup")
@@ -53,15 +60,15 @@ def main():
util.execute_cmd('cp {0}/spot-ingest/ingest_conf.json {1}/spot-ingest/ingest_conf.json'.format(old_spot_path, new_spot_path),log)
log.info("Copy Spot OA configuration files to new Spot location")
- util.execute_cmd('cp {0}/spot-oa/oa/components/iana/iana_config.json {1}/spot-oa/oa/components/iana/iana_config.json'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/oa/components/reputation/reputation_config.json {1}/spot-oa/oa/components/reputation/reputation_config.json'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/oa/components/nc/nc_config.json {1}/spot-oa/oa/components/nc/nc_config.json'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/oa/components/data/engine.json {1}/spot-oa/oa/components/data/engine.json'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/oa/components/iana/iana_config.json {1}/oa/components/iana/iana_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/reputation/reputation_config.json {1}/oa/components/reputation/reputation_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/nc/nc_config.json {1}/oa/components/nc/nc_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/data/engine.json {1}/oa/components/data/engine.json'.format(old_oa_path, new_oa_path),log)
log.info("Copy Spot OA context files to new Spot location")
- util.execute_cmd('cp {0}/spot-oa/context/iploc.csv {1}/spot-oa/context/iploc.csv'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/context/ipranges.csv {1}/spot-oa/context/ipranges.csv'.format(old_spot_path, new_spot_path),log)
- util.execute_cmd('cp {0}/spot-oa/context/networkcontext_1.csv {1}/spot-oa/context/networkcontext_1.csv'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/context/iploc.csv {1}/context/iploc.csv'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/context/ipranges.csv {1}/context/ipranges.csv'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/context/networkcontext_1.csv {1}/context/networkcontext_1.csv'.format(old_oa_path, new_oa_path),log)
log.info("Install browserify and uglifyjs using npm")
os.chdir('{0}/ui'.format(new_oa_path))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bbab8341/spot-setup/migration/create_proxy_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_proxy_migration_tables.hql b/spot-setup/migration/create_proxy_migration_tables.hql
index 56826f4..6eba4f4 100644
--- a/spot-setup/migration/create_proxy_migration_tables.hql
+++ b/spot-setup/migration/create_proxy_migration_tables.hql
@@ -97,3 +97,14 @@ LOCATION '${var:hpath}/proxy/timeline'
TBLPROPERTIES ('skip.header.line.count'='1');
+DROP TABLE IF EXISTS ${var:dbname}.proxy_iana_rcode_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_iana_rcode_tmp (
+respcode STRING,
+respcode_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/proxy/iana_rcode'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bbab8341/spot-setup/migration/drop_proxy_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/drop_proxy_migration_tables.hql b/spot-setup/migration/drop_proxy_migration_tables.hql
index 4849714..858d139 100644
--- a/spot-setup/migration/drop_proxy_migration_tables.hql
+++ b/spot-setup/migration/drop_proxy_migration_tables.hql
@@ -3,3 +3,4 @@ DROP TABLE IF EXISTS ${var:dbname}.proxy_ingest_summary_tmp;
DROP TABLE IF EXISTS ${var:dbname}.proxy_scores_tmp;
DROP TABLE IF EXISTS ${var:dbname}.proxy_storyboard_tmp;
DROP TABLE IF EXISTS ${var:dbname}.proxy_timeline_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.proxy_iana_rcode_tmp;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bbab8341/spot-setup/migration/migrate_old_dns_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.py b/spot-setup/migration/migrate_old_dns_data.py
index f664cb1..15c8741 100755
--- a/spot-setup/migration/migrate_old_dns_data.py
+++ b/spot-setup/migration/migrate_old_dns_data.py
@@ -154,14 +154,20 @@ def main():
dest_table_name = 'dns_edge'
pattern = 'edge*.csv'
edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
- filename = '{0}/{1}'.format(full_day_path, pattern)
- if len(edge_files) > 0:
+ for file in edge_files:
+
+ parts = (re.findall("edge-(\S+).csv", file)[0]).split('_')
+ hh = int(parts[-2])
+ mn = int(parts[-1])
+
+ log.info("Processing File: {0} with HH: {1} and MN: {2}".format(file, hh, mn))
+ filename = '{0}/{1}'.format(full_day_path, file)
load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
util.execute_hive_cmd(load_cmd, log)
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp(regexp_replace(frame_time, '\"', ''), 'MMMMM dd yyyy H:mm:ss.SSS z'), frame_len, ip_dst, ip_src, dns_qry_name, dns_qry_class, dns_qry_type, dns_qry_rcode, dns_a, 0, '', '', '', '' FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp(regexp_replace(frame_time, '\"', ''), 'MMMMM dd yyyy H:mm:ss.SSS z'), frame_len, ip_dst, ip_src, dns_qry_name, '', '0', '0', dns_a, {5}, dns_qry_class, dns_qry_type, dns_qry_rcode, '0' FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, hh, staging_db, staging_table_name)
util.execute_hive_cmd(insert_cmd, log)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/bbab8341/spot-setup/migration/migrate_old_proxy_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.py b/spot-setup/migration/migrate_old_proxy_data.py
index ada7e6c..06ef5f2 100755
--- a/spot-setup/migration/migrate_old_proxy_data.py
+++ b/spot-setup/migration/migrate_old_proxy_data.py
@@ -52,6 +52,7 @@ def main():
util.create_hdfs_folder('{0}/proxy/summary'.format(hdfs_staging_path),log)
util.create_hdfs_folder('{0}/proxy/storyboard'.format(hdfs_staging_path),log)
util.create_hdfs_folder('{0}/proxy/timeline'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/iana_rcode'.format(hdfs_staging_path),log)
util.execute_cmd('hdfs dfs -setfacl -R -m user:impala:rwx {0}'.format(hdfs_staging_path),log)
@@ -143,7 +144,7 @@ def main():
load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
util.execute_hive_cmd(load_cmd, log)
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype, referer, uriport, serverip, scbytes, csbytes, fulluri, 0, '' FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, time, clientip, host, webcat, '0', reqmethod, useragent, resconttype, referer, uriport, serverip, scbytes, csbytes, fulluri, hour(time), respcode FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
util.execute_hive_cmd(insert_cmd, log)
@@ -164,15 +165,23 @@ def main():
##proxy Timeline
log.info("Processing Proxy Timeline")
+
+ log.info("Uploading Proxy IANA resp codes")
+ iana_table_name = 'proxy_iana_rcode_tmp'
+ iana_code_csv = '{0}/oa/components/iana/http-rcode.csv'.format(old_oa_path)
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(iana_code_csv, staging_db, iana_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
staging_table_name = 'proxy_timeline_tmp'
dest_table_name = 'proxy_timeline'
+ pattern = 'timeline*.tsv'
- for filename in fnmatch.filter(os.listdir(full_day_path), 'timeline*.tsv'):
- print filename
+ for file in fnmatch.filter(os.listdir(full_day_path), pattern):
- hash_code = re.findall("timeline-(\S+).tsv", filename)[0]
+ filename = '{0}/{1}'.format(full_day_path, file)
+ hash_code = re.findall("timeline-(\S+).tsv", file)[0]
extsearch_path = "{0}/es-{1}.csv".format(full_day_path, hash_code)
- log.info('File: {0} Hash: {1} Extended Search file: {2}'.format(filename, hash_code, extsearch_path))
+ log.info('File: {0} Hash: {1} Extended Search file: {2}'.format(file, hash_code, extsearch_path))
if os.path.isfile('{0}'.format(extsearch_path)):
@@ -186,7 +195,7 @@ def main():
util.execute_hive_cmd(load_cmd, log)
# Insert into new table from staging table and adding FullURI value
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', tstart, tend, duration, clientip, respcode, '' FROM {6}.{7} where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name)
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', t.tstart, t.tend, t.duration, t.clientip, t.respcode, i.respcode_name FROM {6}.{7} t left join {6}.{8} i on t.respcode = i.respcode where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name, iana_table_name)
util.execute_hive_cmd(insert_cmd, log)
else:
[08/20] incubator-spot git commit: Fixing npm and pip installs
Posted by le...@apache.org.
Fixing npm and pip installs
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/1693f889
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/1693f889
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/1693f889
Branch: refs/heads/SPOT-35_graphql_api
Commit: 1693f889f30476299ce77f21bcce9849a105b190
Parents: 401ffaf
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Thu Jun 29 04:58:54 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/migrate_old_data.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/1693f889/spot-setup/migration/migrate_old_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_data.py b/spot-setup/migration/migrate_old_data.py
index 50ecf6c..d37840b 100755
--- a/spot-setup/migration/migrate_old_data.py
+++ b/spot-setup/migration/migrate_old_data.py
@@ -58,11 +58,11 @@ def main():
log.info("Install browserify and uglifyjs using npm")
os.chdir('{0}/ui'.format(new_oa_path))
- # util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
+ util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
log.info("Install requirements using pip")
os.chdir('{0}'.format(new_oa_path))
- # util.execute_cmd('sudo pip install -r requirements.txt',log)
+ util.execute_cmd('sudo pip install -r requirements.txt',log)
[07/20] incubator-spot git commit: Addind OA context and
configuration migration
Posted by le...@apache.org.
Addind OA context and configuration migration
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/401ffafe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/401ffafe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/401ffafe
Branch: refs/heads/SPOT-35_graphql_api
Commit: 401ffafea3630a97579ab7e79ad30c2f28285c2b
Parents: b829177
Author: Moy Valdovinos <mo...@gmail.com>
Authored: Thu Jun 29 04:49:06 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Tue Jul 4 18:55:44 2017 -0500
----------------------------------------------------------------------
spot-setup/migration/import_ingest_summary.py | 60 -----
spot-setup/migration/import_proxy_timeline.py | 63 ------
spot-setup/migration/migrate_old_data.py | 83 +++++++
spot-setup/migration/migrate_old_dns_data.py | 231 +++++++++++++++++++
spot-setup/migration/migrate_old_dns_data.sh | 195 ----------------
spot-setup/migration/migrate_old_flow_data.py | 232 ++++++++++++++++++++
spot-setup/migration/migrate_old_flow_data.sh | 202 -----------------
spot-setup/migration/migrate_old_proxy_data.py | 222 +++++++++++++++++++
spot-setup/migration/migrate_old_proxy_data.sh | 169 --------------
spot-setup/migration/utilities.py | 89 ++++++++
10 files changed, 857 insertions(+), 689 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/import_ingest_summary.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/import_ingest_summary.py b/spot-setup/migration/import_ingest_summary.py
deleted file mode 100755
index e8fb819..0000000
--- a/spot-setup/migration/import_ingest_summary.py
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import sys
-import re
-import subprocess
-import pandas as pd
-import datetime
-
-filepath = sys.argv[1]
-staging_db_name = sys.argv[2]
-staging_table_name = sys.argv[3]
-dest_db_name = sys.argv[4]
-dest_table_name = sys.argv[5]
-
-df = pd.read_csv(filepath)
-
-s = df.iloc[:,0]
-l_dates = list(s.unique())
-l_dates = map(lambda x: x[0:10].strip(), l_dates)
-l_dates = filter(lambda x: re.match('\d{4}[-/]\d{2}[-/]\d{1}', x), l_dates)
-s_dates = set(l_dates)
-
-for date_str in s_dates:
- date_dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
- print 'Processing day: ', date_str, date_dt.year, date_dt.month, date_dt.day
-
- records = df[df['date'].str.contains(date_str)]
- filename = "ingest_summary_{0}{1}{2}.csv".format(date_dt.year, date_dt.month, date_dt.day)
- records.to_csv(filename, index=False)
-
- insert_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db_name, staging_table_name)
- load_to_table_cmd = "hive -e \"{0}\"".format(insert_cmd)
- print load_to_table_cmd
- subprocess.call(load_to_table_cmd, shell=True)
-
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, total FROM {5}.{6}".format(dest_db_name, dest_table_name, date_dt.year, date_dt.month, date_dt.day, staging_db_name, staging_table_name)
- load_to_table_cmd = "hive -e \"{0}\"".format(insert_cmd)
- print load_to_table_cmd
- subprocess.call(load_to_table_cmd, shell=True)
-
- os.remove(filename)
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/import_proxy_timeline.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/import_proxy_timeline.py b/spot-setup/migration/import_proxy_timeline.py
deleted file mode 100755
index a503c7f..0000000
--- a/spot-setup/migration/import_proxy_timeline.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/env python
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import os
-import sys
-import subprocess
-import pandas as pd
-
-timeline_path = sys.argv[1]
-staging_db_name = sys.argv[2]
-staging_table_name = sys.argv[3]
-dest_db_name = sys.argv[4]
-dest_table_name = sys.argv[5]
-dt = sys.argv[6]
-
-dt_year, dt_month, dt_day = dt.split('-')
-
-print dt_year, dt_month, dt_day
-
-filename = os.path.splitext(os.path.basename(timeline_path))[0]
-hash_code = filename.split('-')[1]
-folder = os.path.dirname(timeline_path)
-extsearch_path = "{0}/es-{1}.csv".format(folder, hash_code)
-
-print filename, hash_code, folder, extsearch_path
-
-if os.path.isfile(extsearch_path):
-
- # Get FullURI from extended search file
- es_df = pd.read_csv(extsearch_path, sep='\t')
- fulluri = es_df.iloc[0]['fulluri']
- print fulluri
-
- # Load timeline to staging table
- insert_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(timeline_path, staging_db_name, staging_table_name)
- load_to_table_cmd = "hive -e \"{0}\"".format(insert_cmd)
- print load_to_table_cmd
- subprocess.call(load_to_table_cmd, shell=True)
-
- # Insert into new table from staging table and adding FullURI value
- insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', tstart, tend, duration, clientip, respcode, '' FROM {6}.{7} where cast(tstart as timestamp) is not null;".format(dest_db_name, dest_table_name, dt_year, dt_month, dt_day, fulluri, staging_db_name, staging_table_name)
- load_to_table_cmd = "hive -e \"{0}\"".format(insert_cmd)
- print load_to_table_cmd
- subprocess.call(load_to_table_cmd, shell=True)
-
-else:
- print "Extended search file {0} doesn't exist. Timeline on file {1} can't be processed".format(extsearch_path, timeline_path)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/migrate_old_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_data.py b/spot-setup/migration/migrate_old_data.py
new file mode 100755
index 0000000..50ecf6c
--- /dev/null
+++ b/spot-setup/migration/migrate_old_data.py
@@ -0,0 +1,83 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+from utilities import util
+
+pipelines=sys.argv[1]
+old_oa_path=sys.argv[2]
+staging_db=sys.argv[3]
+hdfs_staging_path=sys.argv[4]
+dest_db = sys.argv[5]
+impala_daemon = sys.argv[6]
+
+
+# Execution example:
+#./migrate_old_proxy_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Copy ingest_conf.json file from old folder code to new Spot location")
+ util.execute_cmd('cp {0}/spot-ingest/ingest_conf.json {1}/spot-ingest/ingest_conf.json'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Copy Spot OA configuration files to new Spot location")
+ util.execute_cmd('cp {0}/spot-oa/oa/components/iana/iana_config.json {1}/spot-oa/oa/components/iana/iana_config.json'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/oa/components/reputation/reputation_config.json {1}/spot-oa/oa/components/reputation/reputation_config.json'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/oa/components/nc/nc_config.json {1}/spot-oa/oa/components/nc/nc_config.json'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/oa/components/data/engine.json {1}/spot-oa/oa/components/data/engine.json'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Copy Spot OA context files to new Spot location")
+ util.execute_cmd('cp {0}/spot-oa/context/iploc.csv {1}/spot-oa/context/iploc.csv'.format(old_spot_path, new_spot_path),log)
+ util.execute_cmd('cp {0}/spot-oa/context/ipranges.csv {1}/spot-oa/context/ipranges.csv'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Install browserify and uglifyjs using npm")
+ os.chdir('{0}/ui'.format(new_oa_path))
+ # util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
+
+ log.info("Install requirements using pip")
+ os.chdir('{0}'.format(new_oa_path))
+ # util.execute_cmd('sudo pip install -r requirements.txt',log)
+
+
+
+ l_pipelines = pipelines.split(',')
+ valid_pipelines = ('flow', 'dns', 'proxy')
+ if len(l_pipelines) > 0:
+ for pipe in l_pipelines:
+ if pipe in valid_pipelines:
+ os.chdir('{0}/spot-setup/migration'.format(new_spot_path))
+ util.execute_cmd("python migrate_old_{0}_data.py '{1}' '{2}' '{3}' '{4}' '{5}' ".format(pipe, old_oa_path, staging_db, hdfs_staging_path, dest_db, impala_daemon),log)
+ else:
+ log.error("Pipeline {0} is not valid. Valid pipelines are flow, dns or proxy".format(pipe))
+ else:
+ log.error("Pipelines argument must be separated by commas")
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/migrate_old_dns_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.py b/spot-setup/migration/migrate_old_dns_data.py
new file mode 100755
index 0000000..34b792a
--- /dev/null
+++ b/spot-setup/migration/migrate_old_dns_data.py
@@ -0,0 +1,231 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import fnmatch
+import re
+import pandas as pd
+import datetime
+from utilities import util
+
+old_oa_path=sys.argv[1]
+staging_db=sys.argv[2]
+hdfs_staging_path=sys.argv[3]
+dest_db = sys.argv[4]
+impala_daemon = sys.argv[5]
+
+
+# Execution example:
+#./migrate_old_dns_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE.DNS')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Creating HDFS paths for Impala tables")
+ util.create_hdfs_folder('{0}/dns/scores'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/dendro'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/edge'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/summary'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/storyboard'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/threat_dendro'.format(hdfs_staging_path),log)
+ util.execute_cmd('hdfs dfs -setfacl -R -m user:impala:rwx {0}'.format(hdfs_staging_path),log)
+
+
+ log.info("Creating Staging tables in Impala")
+ util.execute_cmd('impala-shell -i {0} --var=hpath={1} --var=dbname={2} -c -f create_dns_migration_tables.hql'.format(impala_daemon, hdfs_staging_path, staging_db),log)
+
+
+ ## dns Ingest Summary
+ log.info('Processing Dns Ingest Summary')
+ ing_sum_path='{0}/data/dns/ingest_summary/'.format(old_oa_path)
+ pattern='is_??????.csv'
+ staging_table_name = 'dns_ingest_summary_tmp'
+ dest_table_name = 'dns_ingest_summary'
+
+ if os.path.exists(ing_sum_path):
+ for file in fnmatch.filter(os.listdir(ing_sum_path), pattern):
+ log.info('Processing file: {0}'.format(file))
+
+ filepath='{0}{1}'.format(ing_sum_path, file)
+ df = pd.read_csv(filepath)
+
+ s = df.iloc[:,0]
+ l_dates = list(s.unique())
+ l_dates = map(lambda x: x[0:10].strip(), l_dates)
+ l_dates = filter(lambda x: re.match('\d{4}[-/]\d{2}[-/]\d{1}', x), l_dates)
+ s_dates = set(l_dates)
+
+ for date_str in s_dates:
+ dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(date_str, dt.year, dt.month, dt.day))
+
+ records = df[df['date'].str.contains(date_str)]
+ filename = "ingest_summary_{0}{1}{2}.csv".format(dt.year, dt.month, dt.day)
+ records.to_csv(filename, index=False)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, total FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ os.remove(filename)
+
+
+ ## Iterating days
+ days_path='{0}/data/dns/'.format(old_oa_path)
+
+ if os.path.exists(days_path):
+ for day_folder in fnmatch.filter(os.listdir(days_path), '2*'):
+
+ print day_folder
+ dt = datetime.datetime.strptime(day_folder, '%Y%m%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(day_folder, dt.year, dt.month, dt.day))
+ full_day_path = '{0}{1}'.format(days_path,day_folder)
+
+
+ ## dns Scores and dns Threat Investigation
+ filename = '{0}/dns_scores.csv'.format(full_day_path)
+ if os.path.isfile(filename):
+
+ log.info("Processing Dns Scores")
+ staging_table_name = 'dns_scores_tmp'
+ dest_table_name = 'dns_scores'
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT frame_time, unix_tstamp, frame_len, ip_dst, dns_qry_name, dns_qry_class, dns_qry_type, dns_qry_rcode, ml_score, tld, query_rep, hh, dns_qry_class_name, dns_qry_type_name, dns_qry_rcode_name, network_context FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ log.info("Processing dns Threat Investigation")
+ staging_table_name = 'dns_scores_tmp'
+ dest_table_name = 'dns_threat_investigation'
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_tstamp, ip_dst, dns_qry_name, ip_sev, dns_sev FROM {5}.{6} WHERE ip_sev > 0 or dns_sev > 0;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ # dns Dendro
+ log.info("Processing Dns Dendro")
+ staging_table_name = 'dns_dendro_tmp'
+ dest_table_name = 'dns_dendro'
+ pattern = 'dendro*.csv'
+ dendro_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ if len(dendro_files) > 0:
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp('{5}', 'yyyyMMMdd'), dns_a, dns_qry_name, ip_dst FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, day_folder, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ## dns Edge
+ log.info("Processing Dns Edge")
+ staging_table_name = 'dns_edge_tmp'
+ dest_table_name = 'dns_edge'
+ pattern = 'edge*.csv'
+ edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ if len(edge_files) > 0:
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp(regexp_replace(frame_time, '\"', ''), 'MMMMM dd yyyy H:mm:ss.SSS z'), frame_len, ip_dst, ip_src, dns_qry_name, dns_qry_class, dns_qry_type, dns_qry_rcode, dns_a, 0, '', '', '', '' FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##dns_storyboard
+ log.info("Processing Dns Storyboard")
+ staging_table_name = 'dns_storyboard_tmp'
+ dest_table_name = 'dns_storyboard'
+ filename = '{0}/threats.csv'.format(full_day_path)
+
+ if os.path.isfile(filename):
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT ip_threat, dns_threat, title, text FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ # dns Threat Dendro
+ log.info("Processing Dns Threat Dendro")
+ staging_table_name = 'dns_threat_dendro_tmp'
+ dest_table_name = 'dns_threat_dendro'
+ pattern = 'threat-dendro*.csv'
+ threat_dendro_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ for file in threat_dendro_files:
+
+ ip = re.findall("threat-dendro-(\S+).csv", file)[0]
+ log.info("Processing File: {0} with IP:{1}".format(file, ip))
+ filename = '{0}/{1}'.format(full_day_path, file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', total, dns_qry_name, ip_dst FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+
+ log.info("Dropping staging tables")
+ util.execute_cmd('impala-shell -i {0} --var=dbname={1} -c -f drop_dns_migration_tables.hql'.format(impala_daemon, staging_db),log)
+
+ log.info("Removing staging tables' path in HDFS")
+ util.execute_cmd('hadoop fs -rm -r {0}/dns/'.format(hdfs_staging_path),log)
+
+ log.info("Moving CSV data to backup folder")
+ util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('mv {0}/data/dns/ {0}/data/backup/'.format(old_oa_path),log)
+
+ log.info("Invalidating metadata in Impala to refresh tables content")
+ util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
+
+
+
+ log.info("Copying advanced mode notebooks to each existing day in new Spot location")
+ for folder in os.listdir('{0}/ipynb/dns/'.format(new_oa_path)):
+ util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb {1}/ipynb/dns/{2}/Advanced_Mode.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+
+ log.info("Copying threat investigation ipynb template to each existing day in new Spot location")
+ for folder in os.listdir('{0}/ipynb/dns/'.format(new_oa_path)):
+ util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb {1}/ipynb/dns/{2}/Threat_Investigation.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/migrate_old_dns_data.sh
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.sh b/spot-setup/migration/migrate_old_dns_data.sh
deleted file mode 100755
index 96341e8..0000000
--- a/spot-setup/migration/migrate_old_dns_data.sh
+++ /dev/null
@@ -1,195 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-OLD_OA_PATH=$1
-NEW_OA_PATH=$2
-STAGING_DB=$3
-HDFS_STAGING_PATH=$4
-DEST_DB=$5
-IMPALA_DEM=$6
-
-# Execution example:
-#./migrate_old_dns_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
-
-hadoop fs -mkdir $HDFS_STAGING_PATH
-hadoop fs -mkdir $HDFS_STAGING_PATH/dns/
-hadoop fs -mkdir $HDFS_STAGING_PATH/dns/scores/
-hadoop fs -mkdir $HDFS_STAGING_PATH/dns/dendro/
-hadoop fs -mkdir $HDFS_STAGING_PATH/dns/edge/
-hadoop fs -mkdir $HDFS_STAGING_PATH/dns/summary/
-hadoop fs -mkdir $HDFS_STAGING_PATH/dns/storyboard/
-hadoop fs -mkdir $HDFS_STAGING_PATH/dns/threat_dendro/
-hdfs dfs -setfacl -R -m user:impala:rwx $HDFS_STAGING_PATH
-
-
-#Creating Staging tables in Impala
-impala-shell -i ${IMPALA_DEM} --var=hpath=${HDFS_STAGING_PATH} --var=dbname=${STAGING_DB} -c -f create_dns_migration_tables.hql
-
-
-## DNS Ingest Summary
-echo "Processing DNS Ingest Summary"
-
-ing_sum_path=$OLD_OA_PATH/data/dns/ingest_summary/is_??????.csv
-
-for file in $ing_sum_path
-do
- echo $file
- ./import_ingest_summary.py "${file}" "${STAGING_DB}" 'dns_ingest_summary_tmp' "${DEST_DB}" 'dns_ingest_summary'
-done
-
-
-DAYS=$OLD_OA_PATH/data/dns/2*
-
-for dir in $DAYS
-do
- day="$(basename $dir)"
- echo "Processing day $day ..."
- y=${day:0:4}
- m=$(expr ${day:4:2} + 0)
- d=$(expr ${day:6:2} + 0)
- echo $y $m $d $d2
- echo $dir
-
-
- ## dns Scores and dns_threat_investigation
- echo "Processing dns Scores"
- if [ -f $dir/dns_scores.csv ]
- then
- command="LOAD DATA LOCAL INPATH '$dir/dns_scores.csv' OVERWRITE INTO TABLE $STAGING_DB.dns_scores_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.dns_scores PARTITION (y=$y, m=$m, d=$d)
-select frame_time, unix_tstamp, frame_len, ip_dst, dns_qry_name, dns_qry_class, dns_qry_type, dns_qry_rcode, ml_score, tld, query_rep, hh, dns_qry_class_name, dns_qry_type_name, dns_qry_rcode_name, network_context
-from $STAGING_DB.dns_scores_tmp;"
- echo $command
- hive -e "$command"
-
- echo "Processing dns Threat Investigation"
- command="INSERT INTO $DEST_DB.dns_threat_investigation PARTITION (y=$y, m=$m, d=$d)
-select unix_tstamp, ip_dst, dns_qry_name, ip_sev, dns_sev
-from $STAGING_DB.dns_scores_tmp
-where ip_sev > 0 or dns_sev > 0;"
- echo $command
- hive -e "$command"
-
- fi
-
- ## dns Dendro
- echo "Processing dns Dendro"
- dendro_files=`ls $dir/dendro*.csv`
-
- if [ ! -z "$dendro_files" ]
- then
-
- command="LOAD DATA LOCAL INPATH '$dir/dendro*.csv' OVERWRITE INTO TABLE $STAGING_DB.dns_dendro_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.dns_dendro PARTITION (y=$y, m=$m, d=$d)
-select unix_timestamp('$y${day:4:2}${day:6:2}', 'yyyyMMMdd'), dns_a, dns_qry_name, ip_dst
-from $STAGING_DB.dns_dendro_tmp;"
- echo $command
- hive -e "$command"
-
- fi
-
- ## dns Edge
- echo "Processing dns Edge"
- edge_files=`ls $dir/edge*.csv`
-
- if [ ! -z "$edge_files" ]
- then
-
- command="LOAD DATA LOCAL INPATH '$dir/edge*.csv' OVERWRITE INTO TABLE $STAGING_DB.dns_edge_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.dns_edge PARTITION (y=$y, m=$m, d=$d)
-select unix_timestamp(regexp_replace(frame_time, '\"', ''), 'MMMMM dd yyyy H:mm:ss.SSS z'), frame_len, ip_dst, ip_src, dns_qry_name, dns_qry_class, dns_qry_type, dns_qry_rcode, dns_a, 0, '', '', '', ''
-from $STAGING_DB.dns_edge_tmp;"
- echo $command
- hive -e "$command"
-
- fi
-
- ##dns_storyboard
- echo "Processing dns Storyboard"
- if [ -f $dir/threats.csv ]
- then
-
- command="LOAD DATA LOCAL INPATH '$dir/threats.csv' OVERWRITE INTO TABLE $STAGING_DB.dns_storyboard_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.dns_storyboard PARTITION (y=$y, m=$m, d=$d)
-select ip_threat, dns_threat, title, text from $STAGING_DB.dns_storyboard_tmp;"
- echo $command
- hive -e "$command"
-
- fi
-
- ##dns_threat_dendro
- echo "Processing dns Threat Dendro"
- threat_dendro_files=`ls $dir/threat-dendro*.csv`
-
- if [ ! -z "$threat_dendro_files" ]
- then
- for file in $threat_dendro_files
- do
- filename="$(basename $file)"
- ip="${filename%.csv}"
- ip="${ip#threat-dendro-}"
- echo $filename $ip
-
- command="LOAD DATA LOCAL INPATH '$file' OVERWRITE INTO TABLE $STAGING_DB.dns_threat_dendro_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.dns_threat_dendro PARTITION (y=$y, m=$m, d=$d)
-select '$ip', total, dns_qry_name, ip_dst
-from $STAGING_DB.dns_threat_dendro_tmp;"
- echo $command
- hive -e "$command"
-
- done
- fi
-done
-
-
-## Copying advanced mode notebooks to new paths in Spot 1.0
-for new_dir in $NEW_OA_PATH/ipynb/dns/*
-do
- cp $OLD_OA_PATH/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb $new_dir/Advanced_Mode.ipynb
-done
-
-# Dropping staging tables
-impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_dns_migration_tables.hql
-
-# Removing staging tables' path in HDFS
-hadoop fs -rm -r $HDFS_STAGING_PATH/dns/
-
-# Moving CSV data to backup folder
-mkdir $OLD_OA_PATH/data/backup/
-mv $OLD_OA_PATH/data/dns/ $OLD_OA_PATH/data/backup/
-
-# Invalidating metadata in Impala to refresh tables content
-impala-shell -i ${IMPALA_DEM} -q "INVALIDATE METADATA;"
-
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/migrate_old_flow_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.py b/spot-setup/migration/migrate_old_flow_data.py
new file mode 100755
index 0000000..eb9253f
--- /dev/null
+++ b/spot-setup/migration/migrate_old_flow_data.py
@@ -0,0 +1,232 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import fnmatch
+import re
+import pandas as pd
+import datetime
+from utilities import util
+
+old_oa_path=sys.argv[1]
+staging_db=sys.argv[2]
+hdfs_staging_path=sys.argv[3]
+dest_db = sys.argv[4]
+impala_daemon = sys.argv[5]
+
+
+# Execution example:
+#./migrate_old_flow_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE.FLOW')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Creating HDFS paths for Impala tables")
+ util.create_hdfs_folder('{0}/flow/scores'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/chords'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/edge'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/summary'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/storyboard'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/threat_investigation'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/timeline'.format(hdfs_staging_path),log)
+ util.execute_cmd('hdfs dfs -setfacl -R -m user:impala:rwx {0}'.format(hdfs_staging_path),log)
+
+
+ log.info("Creating Staging tables in Impala")
+ util.execute_cmd('impala-shell -i {0} --var=hpath={1} --var=dbname={2} -c -f create_flow_migration_tables.hql'.format(impala_daemon, hdfs_staging_path, staging_db),log)
+
+
+ ## Flow Ingest Summary
+ log.info('Processing Flow Ingest Summary')
+ ing_sum_path='{0}/data/flow/ingest_summary/'.format(old_oa_path)
+ pattern='is_??????.csv'
+ staging_table_name = 'flow_ingest_summary_tmp'
+ dest_table_name = 'flow_ingest_summary'
+
+ if os.path.exists(ing_sum_path):
+ for file in fnmatch.filter(os.listdir(ing_sum_path), pattern):
+ log.info('Processing file: {0}'.format(file))
+
+ filepath='{0}{1}'.format(ing_sum_path, file)
+ df = pd.read_csv(filepath)
+
+ s = df.iloc[:,0]
+ l_dates = list(s.unique())
+ l_dates = map(lambda x: x[0:10].strip(), l_dates)
+ l_dates = filter(lambda x: re.match('\d{4}[-/]\d{2}[-/]\d{1}', x), l_dates)
+ s_dates = set(l_dates)
+
+ for date_str in s_dates:
+ dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(date_str, dt.year, dt.month, dt.day))
+
+ records = df[df['date'].str.contains(date_str)]
+ filename = "ingest_summary_{0}{1}{2}.csv".format(dt.year, dt.month, dt.day)
+ records.to_csv(filename, index=False)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, total FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ os.remove(filename)
+
+
+ ## Iterating days
+ days_path='{0}/data/flow/'.format(old_oa_path)
+
+ if os.path.exists(days_path):
+ for day_folder in fnmatch.filter(os.listdir(days_path), '2*'):
+
+ print day_folder
+ dt = datetime.datetime.strptime(day_folder, '%Y%m%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(day_folder, dt.year, dt.month, dt.day))
+ full_day_path = '{0}{1}'.format(days_path,day_folder)
+
+
+ ## Flow Scores and Flow Threat Investigation
+ filename = '{0}/flow_scores.csv'.format(full_day_path)
+ if os.path.isfile(filename):
+
+ log.info("Processing Flow Scores")
+ staging_table_name = 'flow_scores_tmp'
+ dest_table_name = 'flow_scores'
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart,srcip,dstip,sport,dport,proto,ipkt,ibyt,opkt,obyt,score,rank,srcIpInternal,destIpInternal,srcGeo,dstGeo,srcDomain,dstDomain,srcIP_rep,dstIP_rep FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ log.info("Processing Flow Threat Investigation")
+ staging_table_name = 'flow_scores_tmp'
+ dest_table_name = 'flow_threat_investigation'
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart,srcip,dstip,sport,dport,sev FROM {5}.{6} WHERE sev > 0;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ # Flow Chords
+ log.info("Processing Flow Chords")
+ staging_table_name = 'flow_chords_tmp'
+ dest_table_name = 'flow_chords'
+
+ for file in fnmatch.filter(os.listdir(full_day_path), 'chord*.tsv'):
+
+ ip = re.findall("chord-(\S+).tsv", file)[0]
+ ip = ip.replace('_', '.')
+ log.info("Processing File: {0} with IP:{1}".format(file, ip))
+ filename = '{0}/{1}'.format(full_day_path, file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', srcip, dstip, ibyt, ipkt FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ## Flow Edge
+ log.info("Processing Flow Edge")
+ staging_table_name = 'flow_edge_tmp'
+ dest_table_name = 'flow_edge'
+ pattern = 'edge*.tsv'
+ edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ if len(edge_files) > 0:
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart, srcip, dstip, sport, dport, proto, flags, tos, ibyt, ipkt, input, output, rip, obyt, opkt, 0, 0 FROM {5}.{6} WHERE srcip is not NULL;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##flow_storyboard
+ log.info("Processing Flow Storyboard")
+ staging_table_name = 'flow_storyboard_tmp'
+ dest_table_name = 'flow_storyboard'
+ filename = '{0}/threats.csv'.format(full_day_path)
+
+ if os.path.isfile(filename):
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT ip_threat, title, text FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##flow_timeline
+ log.info("Processing Flow Timeline")
+ staging_table_name = 'flow_timeline_tmp'
+ dest_table_name = 'flow_timeline'
+
+ for file in fnmatch.filter(os.listdir(full_day_path), 'sbdet*.tsv'):
+
+ ip = re.findall("sbdet-(\S+).tsv", file)[0]
+ log.info("Processing File: {0} with IP:{1}".format(file, ip))
+ filename = '{0}/{1}'.format(full_day_path, file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', tstart, tend, srcip, dstip, proto, sport, dport, ipkt, ibyt FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+
+ log.info("Dropping staging tables")
+ util.execute_cmd('impala-shell -i {0} --var=dbname={1} -c -f drop_flow_migration_tables.hql'.format(impala_daemon, staging_db),log)
+
+ log.info("Removing staging tables' path in HDFS")
+ util.execute_cmd('hadoop fs -rm -r {0}/flow/'.format(hdfs_staging_path),log)
+
+ log.info("Moving CSV data to backup folder")
+ util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('mv {0}/data/flow/ {0}/data/backup/'.format(old_oa_path),log)
+
+ log.info("Invalidating metadata in Impala to refresh tables content")
+ util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
+
+
+
+ log.info("Copying advanced mode notebooks to each existing day in new Spot location")
+ for folder in os.listdir('{0}/ipynb/flow/'.format(new_oa_path)):
+ util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb {1}/ipynb/flow/{2}/Advanced_Mode.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+
+ log.info("Copying threat investigation ipynb template to each existing day in new Spot location")
+ for folder in os.listdir('{0}/ipynb/flow/'.format(new_oa_path)):
+ util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb {1}/ipynb/flow/{2}/Threat_Investigation.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+
+
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/migrate_old_flow_data.sh
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.sh b/spot-setup/migration/migrate_old_flow_data.sh
deleted file mode 100755
index 9b51987..0000000
--- a/spot-setup/migration/migrate_old_flow_data.sh
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-OLD_OA_PATH=$1
-NEW_OA_PATH=$2
-STAGING_DB=$3
-HDFS_STAGING_PATH=$4
-DEST_DB=$5
-IMPALA_DEM=$6
-
-# Execution example:
-#./migrate_old_flow_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
-
-hadoop fs -mkdir $HDFS_STAGING_PATH
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/scores/
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/chords/
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/edge/
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/summary/
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/storyboard/
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/threat_investigation/
-hadoop fs -mkdir $HDFS_STAGING_PATH/flow/timeline/
-hdfs dfs -setfacl -R -m user:impala:rwx $HDFS_STAGING_PATH
-
-
-#Creating Staging tables in Impala
-impala-shell -i ${IMPALA_DEM} --var=hpath=${HDFS_STAGING_PATH} --var=dbname=${STAGING_DB} -c -f create_flow_migration_tables.hql
-
-
-## Flow Ingest Summary
-echo "Processing Flow Ingest Summary"
-
-ing_sum_path=$OLD_OA_PATH/data/flow/ingest_summary/is_??????.csv
-
-for file in $ing_sum_path
-do
- echo $file
- ./import_ingest_summary.py "${file}" "${STAGING_DB}" 'flow_ingest_summary_tmp' "${DEST_DB}" 'flow_ingest_summary'
-done
-
-
-DAYS=$OLD_OA_PATH/data/flow/2*
-
-for dir in $DAYS
-do
- day="$(basename $dir)"
- echo "Processing day $day ..."
- y=${day:0:4}
- m=$(expr ${day:4:2} + 0)
- d=$(expr ${day:6:2} + 0)
- echo $y $m $d $d2
- echo $dir
-
-
- ## Flow Scores and Flow Threat Investigation
- echo "Processing Flow Scores"
- if [ -f $dir/flow_scores.csv ]
- then
- command="LOAD DATA LOCAL INPATH '$dir/flow_scores.csv' OVERWRITE INTO TABLE $STAGING_DB.flow_scores_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.flow_scores PARTITION (y=$y, m=$m, d=$d)
-select tstart,srcip,dstip,sport,dport,proto,ipkt,ibyt,opkt,obyt,score,rank,srcIpInternal,destIpInternal,srcGeo,dstGeo,srcDomain,dstDomain,srcIP_rep,dstIP_rep
-from $STAGING_DB.flow_scores_tmp;"
- echo $command
- hive -e "$command"
-
- echo "Processing Flow Threat Investigation"
- command="INSERT INTO $DEST_DB.flow_threat_investigation PARTITION (y=$y, m=$m, d=$d)
-select tstart,srcip,dstip,sport,dport,sev
-from $STAGING_DB.flow_scores_tmp
-where sev > 0;"
- echo $command
- hive -e "$command"
-
- fi
-
- ## Flow Chords
- echo "Processing Flow Chords"
- chord_files=`ls $dir/chord*.tsv`
-
- if [ ! -z "$chord_files" ]
- then
- for file in $chord_files
- do
- filename="$(basename $file)"
- ip="${filename%.tsv}"
- ip="${ip#chord-}"
- ip="${ip//_/.}"
- echo $filename $ip
-
- command="LOAD DATA LOCAL INPATH '$file' OVERWRITE INTO TABLE $STAGING_DB.flow_chords_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.flow_chords PARTITION (y=$y, m=$m, d=$d)
- select '$ip', srcip, dstip, ibyt, ipkt from $STAGING_DB.flow_chords_tmp;"
- echo $command
- hive -e "$command"
-
- done
- fi
-
- ## Flow Edge
- echo "Processing Flow Edge"
- edge_files=`ls $dir/edge*.tsv`
-
- if [ ! -z "$edge_files" ]
- then
-
- command="LOAD DATA LOCAL INPATH '$dir/edge*.tsv' OVERWRITE INTO TABLE $STAGING_DB.flow_edge_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.flow_edge PARTITION (y=$y, m=$m, d=$d)
-select tstart, srcip, dstip, sport, dport, proto, flags, tos, ibyt, ipkt, input, output, rip, obyt, opkt, 0, 0
-from $STAGING_DB.flow_edge_tmp
-where srcip is not NULL;"
- echo $command
- hive -e "$command"
-
- fi
-
- ##flow_storyboard
- echo "Processing Flow Storyboard"
- if [ -f $dir/threats.csv ]
- then
-
- command="LOAD DATA LOCAL INPATH '$dir/threats.csv' OVERWRITE INTO TABLE $STAGING_DB.flow_storyboard_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.flow_storyboard PARTITION (y=$y, m=$m, d=$d)
-select ip_threat, title, text from $STAGING_DB.flow_storyboard_tmp;"
- echo $command
- hive -e "$command"
-
- fi
-
- ##flow_timeline
- echo "Processing Flow Timeline"
- timeline_files=`ls $dir/sbdet*.tsv`
-
- if [ ! -z "$timeline_files" ]
- then
- for file in $timeline_files
- do
- filename="$(basename $file)"
- ip="${filename%.tsv}"
- ip="${ip#sbdet-}"
- echo $filename $ip
-
- command="LOAD DATA LOCAL INPATH '$file' OVERWRITE INTO TABLE $STAGING_DB.flow_timeline_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.flow_timeline PARTITION (y=$y, m=$m, d=$d)
-select '$ip', tstart, tend, srcip, dstip, proto, sport, dport, ipkt, ibyt from $STAGING_DB.flow_timeline_tmp;"
- echo $command
- hive -e "$command"
-
- done
- fi
-done
-
-
-## Copying advanced mode notebooks to new paths in Spot 1.0
-for new_dir in $NEW_OA_PATH/ipynb/flow/*
-do
- cp $OLD_OA_PATH/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb $new_dir/Advanced_Mode.ipynb
-done
-
-# Dropping staging tables
-impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_flow_migration_tables.hql
-
-# Removing staging tables' path in HDFS
-hadoop fs -rm -r $HDFS_STAGING_PATH/flow/
-
-# Moving CSV data to backup folder
-mkdir $OLD_OA_PATH/data/backup/
-mv $OLD_OA_PATH/data/flow/ $OLD_OA_PATH/data/backup/
-
-# Invalidating metadata in Impala to refresh tables content
-impala-shell -i ${IMPALA_DEM} -q "INVALIDATE METADATA;"
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/migrate_old_proxy_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.py b/spot-setup/migration/migrate_old_proxy_data.py
new file mode 100755
index 0000000..4298fc6
--- /dev/null
+++ b/spot-setup/migration/migrate_old_proxy_data.py
@@ -0,0 +1,222 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import fnmatch
+import re
+import pandas as pd
+import datetime
+from utilities import util
+
+old_oa_path=sys.argv[1]
+staging_db=sys.argv[2]
+hdfs_staging_path=sys.argv[3]
+dest_db = sys.argv[4]
+impala_daemon = sys.argv[5]
+
+
+# Execution example:
+#./migrate_old_proxy_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE.PROXY')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Creating HDFS paths for Impala tables")
+ util.create_hdfs_folder('{0}/proxy/scores'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/edge'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/summary'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/storyboard'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/timeline'.format(hdfs_staging_path),log)
+ util.execute_cmd('hdfs dfs -setfacl -R -m user:impala:rwx {0}'.format(hdfs_staging_path),log)
+
+
+ log.info("Creating Staging tables in Impala")
+ util.execute_cmd('impala-shell -i {0} --var=hpath={1} --var=dbname={2} -c -f create_proxy_migration_tables.hql'.format(impala_daemon, hdfs_staging_path, staging_db),log)
+
+
+ ## proxy Ingest Summary
+ log.info('Processing proxy Ingest Summary')
+ ing_sum_path='{0}/data/proxy/ingest_summary/'.format(old_oa_path)
+ pattern='is_??????.csv'
+ staging_table_name = 'proxy_ingest_summary_tmp'
+ dest_table_name = 'proxy_ingest_summary'
+
+ if os.path.exists(ing_sum_path):
+ for file in fnmatch.filter(os.listdir(ing_sum_path), pattern):
+ log.info('Processing file: {0}'.format(file))
+
+ filepath='{0}{1}'.format(ing_sum_path, file)
+ df = pd.read_csv(filepath)
+
+ s = df.iloc[:,0]
+ l_dates = list(s.unique())
+ l_dates = map(lambda x: x[0:10].strip(), l_dates)
+ l_dates = filter(lambda x: re.match('\d{4}[-/]\d{2}[-/]\d{1}', x), l_dates)
+ s_dates = set(l_dates)
+
+ for date_str in s_dates:
+ dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(date_str, dt.year, dt.month, dt.day))
+
+ records = df[df['date'].str.contains(date_str)]
+ filename = "ingest_summary_{0}{1}{2}.csv".format(dt.year, dt.month, dt.day)
+ records.to_csv(filename, index=False)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, total FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ os.remove(filename)
+
+
+ ## Iterating days
+ days_path='{0}/data/proxy/'.format(old_oa_path)
+
+ if os.path.exists(days_path):
+ for day_folder in fnmatch.filter(os.listdir(days_path), '2*'):
+
+ print day_folder
+ dt = datetime.datetime.strptime(day_folder, '%Y%m%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(day_folder, dt.year, dt.month, dt.day))
+ full_day_path = '{0}{1}'.format(days_path,day_folder)
+
+
+ ## proxy Scores and proxy Threat Investigation
+ filename = '{0}/proxy_scores.tsv'.format(full_day_path)
+ if os.path.isfile(filename):
+
+ log.info("Processing Proxy Scores")
+ staging_table_name = 'proxy_scores_tmp'
+ dest_table_name = 'proxy_scores'
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, time, clientip, host, reqmethod, useragent, resconttype, duration, username, webcat, referer, respcode, uriport, uripath, uriquery, serverip, scbytes, csbytes, fulluri, word, ml_score, uri_rep, respcode_name, network_context FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ log.info("Processing Proxy Threat Investigation")
+ staging_table_name = 'proxy_scores_tmp'
+ dest_table_name = 'proxy_threat_investigation'
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, fulluri, uri_sev FROM {5}.{6} WHERE uri_sev > 0;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ## proxy Edge
+ log.info("Processing Proxy Edge")
+ staging_table_name = 'proxy_edge_tmp'
+ dest_table_name = 'proxy_edge'
+ pattern = 'edge*.tsv'
+ edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ if len(edge_files) > 0:
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype, referer, uriport, serverip, scbytes, csbytes, fulluri, 0, '' FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##proxy_storyboard
+ log.info("Processing Proxy Storyboard")
+ staging_table_name = 'proxy_storyboard_tmp'
+ dest_table_name = 'proxy_storyboard'
+ filename = '{0}/threats.csv'.format(full_day_path)
+
+ if os.path.isfile(filename):
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT p_threat, title, text FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##proxy Timeline
+ log.info("Processing Proxy Timeline")
+ staging_table_name = 'proxy_timeline_tmp'
+ dest_table_name = 'proxy_timeline'
+
+ for filename in fnmatch.filter(os.listdir(full_day_path), 'timeline*.tsv'):
+ print filename
+
+ hash_code = re.findall("timeline-(\S+).tsv", filename)[0]
+ extsearch_path = "{0}/es-{1}.csv".format(full_day_path, hash_code)
+ log.info('File: {0} Hash: {1} Extended Search file: {2}'.format(filename, hash_code, extsearch_path))
+
+ if os.path.isfile('{0}'.format(extsearch_path)):
+
+ log.info("Getting Full URI from extended search file")
+ es_df = pd.read_csv(extsearch_path, sep='\t')
+ fulluri = es_df.iloc[0]['fulluri']
+ log.info('Full URI found: {0}'.format(fulluri))
+
+ # Load timeline to staging table
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ # Insert into new table from staging table and adding FullURI value
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', tstart, tend, duration, clientip, respcode, '' FROM {6}.{7} where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ else:
+ print "Extended search file {0} doesn't exist. Timeline on file {1} can't be processed".format(extsearch_path, timeline_path)
+
+
+
+ log.info("Dropping staging tables")
+ util.execute_cmd('impala-shell -i {0} --var=dbname={1} -c -f drop_proxy_migration_tables.hql'.format(impala_daemon, staging_db),log)
+
+ log.info("Removing staging tables' path in HDFS")
+ util.execute_cmd('hadoop fs -rm -r {0}/proxy/'.format(hdfs_staging_path),log)
+
+ log.info("Moving CSV data to backup folder")
+ util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('mv {0}/data/proxy/ {0}/data/backup/'.format(old_oa_path),log)
+
+ log.info("Invalidating metadata in Impala to refresh tables content")
+ util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
+
+
+ log.info("Copying advanced mode notebooks to each existing day in new Spot location")
+ for folder in os.listdir('{0}/ipynb/proxy/'.format(new_oa_path)):
+ util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb {1}/ipynb/proxy/{2}/Advanced_Mode.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+
+ log.info("Copying threat investigation ipynb template to each existing day in new Spot location")
+ for folder in os.listdir('{0}/ipynb/proxy/'.format(new_oa_path)):
+ util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb {1}/ipynb/proxy/{2}/Threat_Investigation.ipynb'.format(old_oa_path, new_oa_path, folder),log)
+
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/migrate_old_proxy_data.sh
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.sh b/spot-setup/migration/migrate_old_proxy_data.sh
deleted file mode 100755
index f8eaea3..0000000
--- a/spot-setup/migration/migrate_old_proxy_data.sh
+++ /dev/null
@@ -1,169 +0,0 @@
-#!/bin/bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-OLD_OA_PATH=$1
-NEW_OA_PATH=$2
-STAGING_DB=$3
-HDFS_STAGING_PATH=$4
-DEST_DB=$5
-IMPALA_DEM=$6
-
-# Execution example:
-#./migrate_old_proxy_data.sh '/home/spotuser/incubator-spot_old/spot-oa' '/home/spotuser/incubator-spot_new/spot-oa/' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
-
-hadoop fs -mkdir $HDFS_STAGING_PATH
-hadoop fs -mkdir $HDFS_STAGING_PATH/proxy/
-hadoop fs -mkdir $HDFS_STAGING_PATH/proxy/scores/
-hadoop fs -mkdir $HDFS_STAGING_PATH/proxy/edge/
-hadoop fs -mkdir $HDFS_STAGING_PATH/proxy/summary/
-hadoop fs -mkdir $HDFS_STAGING_PATH/proxy/storyboard/
-hadoop fs -mkdir $HDFS_STAGING_PATH/proxy/timeline/
-hdfs dfs -setfacl -R -m user:impala:rwx $HDFS_STAGING_PATH
-
-
-# Creating Staging tables in Impala
-impala-shell -i ${IMPALA_DEM} --var=hpath=${HDFS_STAGING_PATH} --var=dbname=${STAGING_DB} -c -f create_proxy_migration_tables.hql
-
-
-# Proxy Ingest Summary
-echo "Processing proxy Ingest Summary"
-
-ing_sum_path=$OLD_OA_PATH/data/proxy/ingest_summary/is_??????.csv
-
-for file in $ing_sum_path
-do
- echo $file
- ./import_ingest_summary.py "${file}" "${STAGING_DB}" 'proxy_ingest_summary_tmp' "${DEST_DB}" 'proxy_ingest_summary'
-done
-
-
-DAYS=$OLD_OA_PATH/data/proxy/2*
-
-for dir in $DAYS
-do
-
- day="$(basename $dir)"
- echo "Processing day $day ..."
- y=${day:0:4}
- m=$(expr ${day:4:2} + 0)
- d=$(expr ${day:6:2} + 0)
- echo $y $m $d $d2
- echo $dir
-
-
- ## proxy Scores and proxy_threat_investigation
- echo "Processing proxy Scores"
- if [ -f $dir/proxy_scores.tsv ]
- then
- command="LOAD DATA LOCAL INPATH '$dir/proxy_scores.tsv' OVERWRITE INTO TABLE $STAGING_DB.proxy_scores_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.proxy_scores PARTITION (y=$y, m=$m, d=$d)
-select tdate, time, clientip, host, reqmethod, useragent, resconttype, duration, username, webcat, referer, respcode, uriport, uripath, uriquery, serverip, scbytes, csbytes, fulluri, word, ml_score, uri_rep, respcode_name, network_context
-from $STAGING_DB.proxy_scores_tmp;"
-
- echo $command
- hive -e "$command"
-
- echo "Processing proxy Threat Investigation"
- command="INSERT INTO $DEST_DB.proxy_threat_investigation PARTITION (y=$y, m=$m, d=$d)
-select tdate, fulluri, uri_sev
-from $STAGING_DB.proxy_scores_tmp
-where uri_sev > 0;"
- echo $command
- hive -e "$command"
-
- fi
-
-
- ## proxy Edge
- echo "Processing proxy Edge"
- edge_files=`ls $dir/edge*.tsv`
- #echo $edge_files
- if [ ! -z "$edge_files" ]
- then
-
- command="LOAD DATA LOCAL INPATH '$dir/edge*.tsv' OVERWRITE INTO TABLE $STAGING_DB.proxy_edge_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.proxy_edge PARTITION (y=$y, m=$m, d=$d)
-select tdate, time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype, referer, uriport, serverip, scbytes, csbytes, fulluri, 0, ''
-from $STAGING_DB.proxy_edge_tmp;"
- echo $command
- hive -e "$command"
-
- fi
-
-
- ##proxy_storyboard
- echo "Processing proxy Storyboard"
- if [ -f $dir/threats.csv ]
- then
-
- command="LOAD DATA LOCAL INPATH '$dir/threats.csv' OVERWRITE INTO TABLE $STAGING_DB.proxy_storyboard_tmp;"
- echo $command
- hive -e "$command"
-
- command="INSERT INTO $DEST_DB.proxy_storyboard PARTITION (y=$y, m=$m, d=$d)
-select p_threat, title, text
-from $STAGING_DB.proxy_storyboard_tmp;"
- echo $command
- hive -e "$command"
-
- fi
-
- ##proxy_timeline
- echo "Processing proxy Timeline"
- timeline_files=`ls $dir/timeline*.tsv`
-
- if [ ! -z "$timeline_files" ]
- then
- for file in $timeline_files
- do
-
- echo $file
- ./import_proxy_timeline.py "${file}" "${STAGING_DB}" 'proxy_timeline_tmp' "${DEST_DB}" 'proxy_timeline' "${y}-${m}-${d}"
-
- done
- fi
-done
-
-
-## Copying advanced mode notebooks to new paths in Spot 1.0
-for new_dir in $NEW_OA_PATH/ipynb/proxy/*
-do
- cp $OLD_OA_PATH/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb $new_dir/Advanced_Mode.ipynb
-done
-
-# Dropping staging tables
-impala-shell -i ${IMPALA_DEM} --var=dbname=${STAGING_DB} -c -f drop_proxy_migration_tables.hql
-
-# Removing staging tables' path in HDFS
-hadoop fs -rm -r $HDFS_STAGING_PATH/proxy/
-
-# Moving CSV data to backup folder
-mkdir $OLD_OA_PATH/data/backup/
-mv $OLD_OA_PATH/data/proxy/ $OLD_OA_PATH/data/backup/
-
-# Invalidating metadata in Impala to refresh tables content
-impala-shell -i ${IMPALA_DEM} -q "INVALIDATE METADATA;"
-
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/401ffafe/spot-setup/migration/utilities.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/utilities.py b/spot-setup/migration/utilities.py
new file mode 100644
index 0000000..046679e
--- /dev/null
+++ b/spot-setup/migration/utilities.py
@@ -0,0 +1,89 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import logging
+
+
+class util(object):
+
+ @classmethod
+ def create_hdfs_folder(cls,hdfs_path,logger):
+ hadoop_create_folder="hadoop fs -mkdir -p {0}".format(hdfs_path)
+ logger.info("SPOT.Utils: Creating hdfs folder: {0}".format(hadoop_create_folder))
+ subprocess.call(hadoop_create_folder,shell=True)
+
+ @classmethod
+ def load_to_hdfs(cls,file_local_path,file_hdfs_path,logger):
+ # move file to hdfs.
+ load_to_hadoop_script = "hadoop fs -moveFromLocal {0} {1}".format(file_local_path,file_hdfs_path)
+ logger.info("SPOT.Utils: Loading file to hdfs: {0}".format(load_to_hadoop_script))
+ subprocess.call(load_to_hadoop_script,shell=True)
+
+ @classmethod
+ def get_logger(cls,logger_name,create_file=False):
+
+ # create logger for prd_ci
+ log = logging.getLogger(logger_name)
+ log.setLevel(level=logging.INFO)
+
+ # create formatter and add it to the handlers
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ if create_file:
+ # create file handler for logger.
+ fh = logging.FileHandler('SPOT.log')
+ fh.setLevel(level=logging.DEBUG)
+ fh.setFormatter(formatter)
+ # reate console handler for logger.
+ ch = logging.StreamHandler()
+ ch.setLevel(level=logging.DEBUG)
+ ch.setFormatter(formatter)
+
+ # add handlers to logger.
+ if create_file:
+ log.addHandler(fh)
+
+ log.addHandler(ch)
+ return log
+
+ @classmethod
+ def execute_cmd(cls,command,logger):
+
+ try:
+ logger.info("SPOT.Utils: Executing: {0}".format(command))
+ subprocess.call(command,shell=True)
+
+ except subprocess.CalledProcessError as e:
+ logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
+ sys.exit(1)
+
+ @classmethod
+ def execute_hive_cmd(cls,command,logger):
+
+ try:
+ logger.info("SPOT.Utils: Executing hive command: hive -e \"{0}\"".format(command))
+ subprocess.call("hive -e \"{0}\"".format(command),shell=True)
+
+ except subprocess.CalledProcessError as e:
+ logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
+ sys.exit(1)
+