You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 21:37:07 UTC
[01/18] incubator-spot git commit: syncing with master
Repository: incubator-spot
Updated Branches:
refs/heads/SPOT-181_ODM 2bf696efd -> fbcfa6835
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/drop_dns_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/drop_dns_migration_tables.hql b/spot-setup/migration/drop_dns_migration_tables.hql
new file mode 100644
index 0000000..a156bdd
--- /dev/null
+++ b/spot-setup/migration/drop_dns_migration_tables.hql
@@ -0,0 +1,23 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_dendro_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.dns_edge_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.dns_ingest_summary_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.dns_scores_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.dns_storyboard_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.dns_threat_dendro_tmp;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/drop_flow_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/drop_flow_migration_tables.hql b/spot-setup/migration/drop_flow_migration_tables.hql
new file mode 100644
index 0000000..6570fec
--- /dev/null
+++ b/spot-setup/migration/drop_flow_migration_tables.hql
@@ -0,0 +1,23 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_scores_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.flow_chords_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.flow_edge_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.flow_ingest_summary_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.flow_storyboard_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.flow_timeline_tmp;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/drop_proxy_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/drop_proxy_migration_tables.hql b/spot-setup/migration/drop_proxy_migration_tables.hql
new file mode 100644
index 0000000..7ae3c54
--- /dev/null
+++ b/spot-setup/migration/drop_proxy_migration_tables.hql
@@ -0,0 +1,23 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_edge_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.proxy_ingest_summary_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.proxy_scores_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.proxy_storyboard_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.proxy_timeline_tmp;
+DROP TABLE IF EXISTS ${var:dbname}.proxy_iana_rcode_tmp;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/migrate_old_dns_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_dns_data.py b/spot-setup/migration/migrate_old_dns_data.py
new file mode 100755
index 0000000..fb7dca8
--- /dev/null
+++ b/spot-setup/migration/migrate_old_dns_data.py
@@ -0,0 +1,248 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import fnmatch
+import re
+import pandas as pd
+import datetime
+from utilities import util
+
+old_oa_path=sys.argv[1]
+staging_db=sys.argv[2]
+hdfs_staging_path=sys.argv[3]
+dest_db = sys.argv[4]
+impala_daemon = sys.argv[5]
+
+
+# Execution example:
+#./migrate_old_dns_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE.DNS')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Creating HDFS paths for Impala tables")
+ util.create_hdfs_folder('{0}/dns/scores'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/dendro'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/edge'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/summary'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/storyboard'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/dns/threat_dendro'.format(hdfs_staging_path),log)
+ util.execute_cmd('hdfs dfs -setfacl -R -m user:impala:rwx {0}'.format(hdfs_staging_path),log)
+
+
+ log.info("Creating Staging tables in Impala")
+ util.execute_cmd('impala-shell -i {0} --var=hpath={1} --var=dbname={2} -c -f create_dns_migration_tables.hql'.format(impala_daemon, hdfs_staging_path, staging_db),log)
+
+
+ ## dns Ingest Summary
+ log.info('Processing Dns Ingest Summary')
+ ing_sum_path='{0}/data/dns/ingest_summary/'.format(old_oa_path)
+ pattern='is_??????.csv'
+ staging_table_name = 'dns_ingest_summary_tmp'
+ dest_table_name = 'dns_ingest_summary'
+
+ if os.path.exists(ing_sum_path):
+ for file in fnmatch.filter(os.listdir(ing_sum_path), pattern):
+ log.info('Processing file: {0}'.format(file))
+
+ filepath='{0}{1}'.format(ing_sum_path, file)
+ df = pd.read_csv(filepath)
+
+ s = df.iloc[:,0]
+ l_dates = list(s.unique())
+ l_dates = map(lambda x: x[0:10].strip(), l_dates)
+ l_dates = filter(lambda x: re.match('\d{4}[-/]\d{2}[-/]\d{1}', x), l_dates)
+ s_dates = set(l_dates)
+
+ for date_str in s_dates:
+ dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(date_str, dt.year, dt.month, dt.day))
+
+ records = df[df['date'].str.contains(date_str)]
+ filename = "ingest_summary_{0}{1}{2}.csv".format(dt.year, dt.month, dt.day)
+ records.to_csv(filename, index=False)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, total FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ os.remove(filename)
+
+
+ ## Iterating days
+ days_path='{0}/data/dns/'.format(old_oa_path)
+
+ if os.path.exists(days_path):
+ for day_folder in fnmatch.filter(os.listdir(days_path), '2*'):
+
+ print day_folder
+ dt = datetime.datetime.strptime(day_folder, '%Y%m%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(day_folder, dt.year, dt.month, dt.day))
+ full_day_path = '{0}{1}'.format(days_path,day_folder)
+
+
+ ## dns Scores and dns Threat Investigation
+ filename = '{0}/dns_scores.csv'.format(full_day_path)
+ if os.path.isfile(filename):
+
+ log.info("Processing Dns Scores")
+ staging_table_name = 'dns_scores_tmp'
+ dest_table_name = 'dns_scores'
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT frame_time, unix_tstamp, frame_len, ip_dst, dns_qry_name, dns_qry_class, dns_qry_type, dns_qry_rcode, ml_score, tld, query_rep, hh, dns_qry_class_name, dns_qry_type_name, dns_qry_rcode_name, network_context FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ log.info("Processing dns Threat Investigation")
+ staging_table_name = 'dns_scores_tmp'
+ dest_table_name = 'dns_threat_investigation'
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_tstamp, ip_dst, dns_qry_name, ip_sev, dns_sev FROM {5}.{6} WHERE ip_sev > 0 or dns_sev > 0;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ # dns Dendro
+ log.info("Processing Dns Dendro")
+ staging_table_name = 'dns_dendro_tmp'
+ dest_table_name = 'dns_dendro'
+ pattern = 'dendro*.csv'
+ dendro_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ if len(dendro_files) > 0:
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp('{5}', 'yyyyMMMdd'), dns_a, dns_qry_name, ip_dst FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, day_folder, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ## dns Edge
+ log.info("Processing Dns Edge")
+ staging_table_name = 'dns_edge_tmp'
+ dest_table_name = 'dns_edge'
+ pattern = 'edge*.csv'
+ edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+
+ for file in edge_files:
+
+ parts = (re.findall("edge-(\S+).csv", file)[0]).split('_')
+ hh = int(parts[-2])
+ mn = int(parts[-1])
+
+ log.info("Processing File: {0} with HH: {1} and MN: {2}".format(file, hh, mn))
+
+ log.info("Removing double quotes File: {0}".format(file))
+ fixed_file = '{0}.fixed'.format(file)
+ sed_cmd = "sed 's/\"//g' {0}/{1} > {0}/{2}".format(full_day_path, file, fixed_file)
+ util.execute_cmd(sed_cmd, log)
+ filename = '{0}/{1}'.format(full_day_path, fixed_file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT unix_timestamp(frame_time, 'MMMMM dd yyyy H:mm:ss.SSS z'), frame_len, ip_dst, ip_src, dns_qry_name, '', '0', '0', dns_a, {5}, dns_qry_class, dns_qry_type, dns_qry_rcode, '0' FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, hh, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ os.remove(filename)
+
+
+ ##dns_storyboard
+ log.info("Processing Dns Storyboard")
+ staging_table_name = 'dns_storyboard_tmp'
+ dest_table_name = 'dns_storyboard'
+ filename = '{0}/threats.csv'.format(full_day_path)
+
+ if os.path.isfile(filename):
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT ip_threat, dns_threat, title, text FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ # dns Threat Dendro
+ log.info("Processing Dns Threat Dendro")
+ staging_table_name = 'dns_threat_dendro_tmp'
+ dest_table_name = 'dns_threat_dendro'
+ pattern = 'threat-dendro*.csv'
+ threat_dendro_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ for file in threat_dendro_files:
+
+ ip = re.findall("threat-dendro-(\S+).csv", file)[0]
+ log.info("Processing File: {0} with IP:{1}".format(file, ip))
+ filename = '{0}/{1}'.format(full_day_path, file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', total, dns_qry_name, ip_dst FROM {6}.{7} WHERE dns_qry_name is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+
+ log.info("Dropping staging tables")
+ util.execute_cmd('impala-shell -i {0} --var=dbname={1} -c -f drop_dns_migration_tables.hql'.format(impala_daemon, staging_db),log)
+
+ log.info("Removing staging tables' path in HDFS")
+ util.execute_cmd('hadoop fs -rm -r {0}/dns/'.format(hdfs_staging_path),log)
+
+ log.info("Moving CSV data to backup folder")
+ util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('cp -r {0}/data/dns/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('rm -r {0}/data/dns/'.format(old_oa_path),log)
+
+
+ log.info("Invalidating metadata in Impala to refresh tables content")
+ util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
+
+
+ log.info("Creating ipynb template structure and copying advanced mode and threat investigation ipynb templates for each pre-existing day in the new Spot location")
+ ipynb_pipeline_path = '{0}/ipynb/dns/'.format(old_oa_path)
+ if os.path.exists(ipynb_pipeline_path):
+ for folder in os.listdir(ipynb_pipeline_path):
+ log.info("Creating ipynb dns folders in new Spot locaiton: {0}".format(folder))
+ util.execute_cmd('mkdir -p {0}/ipynb/dns/{1}/'.format(new_oa_path, folder),log)
+ log.info("Copying advanced mode ipynb template")
+ util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb {0}/ipynb/dns/{1}/Advanced_Mode.ipynb'.format(new_oa_path, folder),log)
+ log.info("Copying threat investigation ipynb template")
+ util.execute_cmd('cp {0}/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb {0}/ipynb/dns/{1}/Threat_Investigation.ipynb'.format(new_oa_path, folder),log)
+
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/migrate_old_flow_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_flow_data.py b/spot-setup/migration/migrate_old_flow_data.py
new file mode 100755
index 0000000..3b5543b
--- /dev/null
+++ b/spot-setup/migration/migrate_old_flow_data.py
@@ -0,0 +1,240 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import fnmatch
+import re
+import pandas as pd
+import datetime
+from utilities import util
+
+old_oa_path=sys.argv[1]
+staging_db=sys.argv[2]
+hdfs_staging_path=sys.argv[3]
+dest_db = sys.argv[4]
+impala_daemon = sys.argv[5]
+
+
+# Execution example:
+#./migrate_old_flow_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE.FLOW')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Creating HDFS paths for Impala tables")
+ util.create_hdfs_folder('{0}/flow/scores'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/chords'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/edge'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/summary'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/storyboard'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/threat_investigation'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/flow/timeline'.format(hdfs_staging_path),log)
+ util.execute_cmd('hdfs dfs -setfacl -R -m user:impala:rwx {0}'.format(hdfs_staging_path),log)
+
+
+ log.info("Creating Staging tables in Impala")
+ util.execute_cmd('impala-shell -i {0} --var=hpath={1} --var=dbname={2} -c -f create_flow_migration_tables.hql'.format(impala_daemon, hdfs_staging_path, staging_db),log)
+
+
+ ## Flow Ingest Summary
+ log.info('Processing Flow Ingest Summary')
+ ing_sum_path='{0}/data/flow/ingest_summary/'.format(old_oa_path)
+ pattern='is_??????.csv'
+ staging_table_name = 'flow_ingest_summary_tmp'
+ dest_table_name = 'flow_ingest_summary'
+
+ if os.path.exists(ing_sum_path):
+ for file in fnmatch.filter(os.listdir(ing_sum_path), pattern):
+ log.info('Processing file: {0}'.format(file))
+
+ filepath='{0}{1}'.format(ing_sum_path, file)
+ df = pd.read_csv(filepath)
+
+ s = df.iloc[:,0]
+ l_dates = list(s.unique())
+ l_dates = map(lambda x: x[0:10].strip(), l_dates)
+ l_dates = filter(lambda x: re.match('\d{4}[-/]\d{2}[-/]\d{1}', x), l_dates)
+ s_dates = set(l_dates)
+
+ for date_str in s_dates:
+ dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(date_str, dt.year, dt.month, dt.day))
+
+ records = df[df['date'].str.contains(date_str)]
+ filename = "ingest_summary_{0}{1}{2}.csv".format(dt.year, dt.month, dt.day)
+ records.to_csv(filename, index=False)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, total FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ os.remove(filename)
+
+
+ ## Iterating days
+ days_path='{0}/data/flow/'.format(old_oa_path)
+
+ if os.path.exists(days_path):
+ for day_folder in fnmatch.filter(os.listdir(days_path), '2*'):
+
+ print day_folder
+ dt = datetime.datetime.strptime(day_folder, '%Y%m%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(day_folder, dt.year, dt.month, dt.day))
+ full_day_path = '{0}{1}'.format(days_path,day_folder)
+
+
+ ## Flow Scores and Flow Threat Investigation
+ filename = '{0}/flow_scores.csv'.format(full_day_path)
+ if os.path.isfile(filename):
+
+ log.info("Processing Flow Scores")
+ staging_table_name = 'flow_scores_tmp'
+ dest_table_name = 'flow_scores'
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart,srcip,dstip,sport,dport,proto,ipkt,ibyt,opkt,obyt,score,rank,srcIpInternal,destIpInternal,srcGeo,dstGeo,srcDomain,dstDomain,srcIP_rep,dstIP_rep FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ log.info("Processing Flow Threat Investigation")
+ staging_table_name = 'flow_scores_tmp'
+ dest_table_name = 'flow_threat_investigation'
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart,srcip,dstip,sport,dport,sev FROM {5}.{6} WHERE sev > 0;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ # Flow Chords
+ log.info("Processing Flow Chords")
+ staging_table_name = 'flow_chords_tmp'
+ dest_table_name = 'flow_chords'
+
+ for file in fnmatch.filter(os.listdir(full_day_path), 'chord*.tsv'):
+
+ ip = re.findall("chord-(\S+).tsv", file)[0]
+ ip = ip.replace('_', '.')
+ log.info("Processing File: {0} with IP:{1}".format(file, ip))
+ filename = '{0}/{1}'.format(full_day_path, file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', srcip, dstip, ibyt, ipkt FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ## Flow Edge
+ log.info("Processing Flow Edge")
+ staging_table_name = 'flow_edge_tmp'
+ dest_table_name = 'flow_edge'
+ pattern = 'edge*.tsv'
+ edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+
+ for file in edge_files:
+
+ parts = (re.findall("edge-(\S+).tsv", file)[0]).split('-')
+ hh = int(parts[2])
+ mn = int(parts[3])
+
+ log.info("Processing File: {0} with HH: {1} and MN: {2}".format(file, hh, mn))
+ filename = '{0}/{1}'.format(full_day_path, file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tstart, srcip, dstip, sport, dport, proto, flags, tos, ibyt, ipkt, input, output, rip, obyt, opkt, {5}, {6} FROM {7}.{8} WHERE srcip is not NULL;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, hh, mn, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##flow_storyboard
+ log.info("Processing Flow Storyboard")
+ staging_table_name = 'flow_storyboard_tmp'
+ dest_table_name = 'flow_storyboard'
+ filename = '{0}/threats.csv'.format(full_day_path)
+
+ if os.path.isfile(filename):
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT ip_threat, title, text FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##flow_timeline
+ log.info("Processing Flow Timeline")
+ staging_table_name = 'flow_timeline_tmp'
+ dest_table_name = 'flow_timeline'
+
+ for file in fnmatch.filter(os.listdir(full_day_path), 'sbdet*.tsv'):
+
+ ip = re.findall("sbdet-(\S+).tsv", file)[0]
+ log.info("Processing File: {0} with IP:{1}".format(file, ip))
+ filename = '{0}/{1}'.format(full_day_path, file)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', tstart, tend, srcip, dstip, proto, sport, dport, ipkt, ibyt FROM {6}.{7};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, ip, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+
+ log.info("Dropping staging tables")
+ util.execute_cmd('impala-shell -i {0} --var=dbname={1} -c -f drop_flow_migration_tables.hql'.format(impala_daemon, staging_db),log)
+
+ log.info("Removing staging tables' path in HDFS")
+ util.execute_cmd('hadoop fs -rm -r {0}/flow/'.format(hdfs_staging_path),log)
+
+ log.info("Moving CSV data to backup folder")
+ util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('cp -r {0}/data/flow/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('rm -r {0}/data/flow/'.format(old_oa_path),log)
+
+ log.info("Invalidating metadata in Impala to refresh tables content")
+ util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
+
+
+ log.info("Creating ipynb template structure and copying advanced mode and threat investigation ipynb templates for each pre-existing day in the new Spot location")
+ ipynb_pipeline_path = '{0}/ipynb/flow/'.format(old_oa_path)
+ if os.path.exists(ipynb_pipeline_path):
+ for folder in os.listdir(ipynb_pipeline_path):
+ log.info("Creating ipynb flow folders in new Spot locaiton: {0}".format(folder))
+ util.execute_cmd('mkdir -p {0}/ipynb/flow/{1}/'.format(new_oa_path, folder),log)
+ log.info("Copying advanced mode ipynb template")
+ util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb {0}/ipynb/flow/{1}/Advanced_Mode.ipynb'.format(new_oa_path, folder),log)
+ log.info("Copying threat investigation ipynb template")
+ util.execute_cmd('cp {0}/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb {0}/ipynb/flow/{1}/Threat_Investigation.ipynb'.format(new_oa_path, folder),log)
+
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/migrate_old_proxy_data.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/migrate_old_proxy_data.py b/spot-setup/migration/migrate_old_proxy_data.py
new file mode 100755
index 0000000..51ef7b2
--- /dev/null
+++ b/spot-setup/migration/migrate_old_proxy_data.py
@@ -0,0 +1,236 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import fnmatch
+import re
+import pandas as pd
+import datetime
+from utilities import util
+
+old_oa_path=sys.argv[1]
+staging_db=sys.argv[2]
+hdfs_staging_path=sys.argv[3]
+dest_db = sys.argv[4]
+impala_daemon = sys.argv[5]
+
+
+# Execution example:
+#./migrate_old_proxy_data.py '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE.PROXY')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+ new_spot_path = os.path.split(os.path.split(cur_path)[0])[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+ log.info('New Spot OA path: {0}'.format(new_oa_path))
+ old_spot_path = os.path.split(old_oa_path)[0]
+
+
+ log.info("Creating HDFS paths for Impala tables")
+ util.create_hdfs_folder('{0}/proxy/scores'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/edge'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/summary'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/storyboard'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/timeline'.format(hdfs_staging_path),log)
+ util.create_hdfs_folder('{0}/proxy/iana_rcode'.format(hdfs_staging_path),log)
+ util.execute_cmd('hdfs dfs -setfacl -R -m user:impala:rwx {0}'.format(hdfs_staging_path),log)
+
+
+ log.info("Creating Staging tables in Impala")
+ util.execute_cmd('impala-shell -i {0} --var=hpath={1} --var=dbname={2} -c -f create_proxy_migration_tables.hql'.format(impala_daemon, hdfs_staging_path, staging_db),log)
+
+
+ ## proxy Ingest Summary
+ log.info('Processing proxy Ingest Summary')
+ ing_sum_path='{0}/data/proxy/ingest_summary/'.format(old_oa_path)
+ pattern='is_??????.csv'
+ staging_table_name = 'proxy_ingest_summary_tmp'
+ dest_table_name = 'proxy_ingest_summary'
+
+ if os.path.exists(ing_sum_path):
+ for file in fnmatch.filter(os.listdir(ing_sum_path), pattern):
+ log.info('Processing file: {0}'.format(file))
+
+ filepath='{0}{1}'.format(ing_sum_path, file)
+ df = pd.read_csv(filepath)
+
+ s = df.iloc[:,0]
+ l_dates = list(s.unique())
+ l_dates = map(lambda x: x[0:10].strip(), l_dates)
+ l_dates = filter(lambda x: re.match('\d{4}[-/]\d{2}[-/]\d{1}', x), l_dates)
+ s_dates = set(l_dates)
+
+ for date_str in s_dates:
+ dt = datetime.datetime.strptime(date_str, '%Y-%m-%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(date_str, dt.year, dt.month, dt.day))
+
+ records = df[df['date'].str.contains(date_str)]
+ filename = "ingest_summary_{0}{1}{2}.csv".format(dt.year, dt.month, dt.day)
+ records.to_csv(filename, index=False)
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, total FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ os.remove(filename)
+
+
+ ## Iterating days
+ days_path='{0}/data/proxy/'.format(old_oa_path)
+
+ if os.path.exists(days_path):
+ for day_folder in fnmatch.filter(os.listdir(days_path), '2*'):
+
+ print day_folder
+ dt = datetime.datetime.strptime(day_folder, '%Y%m%d')
+ log.info('Processing day: {0} {1} {2} {3}'.format(day_folder, dt.year, dt.month, dt.day))
+ full_day_path = '{0}{1}'.format(days_path,day_folder)
+
+
+ ## proxy Scores and proxy Threat Investigation
+ filename = '{0}/proxy_scores.tsv'.format(full_day_path)
+ if os.path.isfile(filename):
+
+ log.info("Processing Proxy Scores")
+ staging_table_name = 'proxy_scores_tmp'
+ dest_table_name = 'proxy_scores'
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, time, clientip, host, reqmethod, useragent, resconttype, duration, username, webcat, referer, respcode, uriport, uripath, uriquery, serverip, scbytes, csbytes, fulluri, word, ml_score, uri_rep, respcode_name, network_context FROM {5}.{6}".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ log.info("Processing Proxy Threat Investigation")
+ staging_table_name = 'proxy_scores_tmp'
+ dest_table_name = 'proxy_threat_investigation'
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT tdate, fulluri, uri_sev FROM {5}.{6} WHERE uri_sev > 0;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ## proxy IANA resp codes
+ log.info("Uploading Proxy IANA resp codes")
+ iana_table_name = 'proxy_iana_rcode_tmp'
+ iana_code_csv = '{0}/oa/components/iana/http-rcode.csv'.format(old_oa_path)
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(iana_code_csv, staging_db, iana_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+
+ ## proxy Edge
+ log.info("Processing Proxy Edge")
+ staging_table_name = 'proxy_edge_tmp'
+ dest_table_name = 'proxy_edge'
+ pattern = 'edge*.tsv'
+ edge_files = fnmatch.filter(os.listdir(full_day_path), pattern)
+ filename = '{0}/{1}'.format(full_day_path, pattern)
+
+ if len(edge_files) > 0:
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT e.tdate, e.time, e.clientip, e.host, e.webcat, COALESCE(i.respcode,'0'), e.reqmethod, e.useragent, e.resconttype, e.referer, e.uriport, e.serverip, e.scbytes, e.csbytes, e.fulluri, hour(e.time), e.respcode FROM {5}.{6} e left join {5}.{7} i on e.respcode = i.respcode_name ;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name, iana_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##proxy_storyboard
+ log.info("Processing Proxy Storyboard")
+ staging_table_name = 'proxy_storyboard_tmp'
+ dest_table_name = 'proxy_storyboard'
+ filename = '{0}/threats.csv'.format(full_day_path)
+
+ if os.path.isfile(filename):
+
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT p_threat, title, text FROM {5}.{6};".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, staging_db, staging_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+
+ ##proxy Timeline
+ log.info("Processing Proxy Timeline")
+ staging_table_name = 'proxy_timeline_tmp'
+ dest_table_name = 'proxy_timeline'
+ pattern = 'timeline*.tsv'
+
+ for file in fnmatch.filter(os.listdir(full_day_path), pattern):
+
+ filename = '{0}/{1}'.format(full_day_path, file)
+ hash_code = re.findall("timeline-(\S+).tsv", file)[0]
+ extsearch_path = "{0}/es-{1}.csv".format(full_day_path, hash_code)
+ log.info('File: {0} Hash: {1} Extended Search file: {2}'.format(file, hash_code, extsearch_path))
+
+ if os.path.isfile('{0}'.format(extsearch_path)):
+
+ log.info("Getting Full URI from extended search file")
+ es_df = pd.read_csv(extsearch_path, sep='\t')
+ fulluri = es_df.iloc[0]['fulluri']
+ log.info('Full URI found: {0}'.format(fulluri))
+
+ # Load timeline to staging table
+ load_cmd = "LOAD DATA LOCAL INPATH '{0}' OVERWRITE INTO TABLE {1}.{2};".format(filename, staging_db, staging_table_name)
+ util.execute_hive_cmd(load_cmd, log)
+
+ # Insert into new table from staging table and adding FullURI value
+ insert_cmd = "INSERT INTO {0}.{1} PARTITION (y={2}, m={3}, d={4}) SELECT '{5}', t.tstart, t.tend, t.duration, t.clientip, t.respcode, COALESCE(i.respcode_name,'') FROM {6}.{7} t left join {6}.{8} i on t.respcode = i.respcode where cast(tstart as timestamp) is not null;".format(dest_db, dest_table_name, dt.year, dt.month, dt.day, fulluri, staging_db, staging_table_name, iana_table_name)
+ util.execute_hive_cmd(insert_cmd, log)
+
+ else:
+ print "Extended search file {0} doesn't exist. Timeline on file {1} can't be processed".format(extsearch_path, timeline_path)
+
+
+
+ log.info("Dropping staging tables")
+ util.execute_cmd('impala-shell -i {0} --var=dbname={1} -c -f drop_proxy_migration_tables.hql'.format(impala_daemon, staging_db),log)
+
+ log.info("Removing staging tables' path in HDFS")
+ util.execute_cmd('hadoop fs -rm -r {0}/proxy/'.format(hdfs_staging_path),log)
+
+ log.info("Moving CSV data to backup folder")
+ util.execute_cmd('mkdir {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('cp -r {0}/data/proxy/ {0}/data/backup/'.format(old_oa_path),log)
+ util.execute_cmd('rm -r {0}/data/proxy/'.format(old_oa_path),log)
+
+ log.info("Invalidating metadata in Impala to refresh tables content")
+ util.execute_cmd('impala-shell -i {0} -q "INVALIDATE METADATA;"'.format(impala_daemon),log)
+
+
+ log.info("Creating ipynb template structure and copying advanced mode and threat investigation ipynb templates for each pre-existing day in the new Spot location")
+ ipynb_pipeline_path = '{0}/ipynb/proxy/'.format(old_oa_path)
+ if os.path.exists(ipynb_pipeline_path):
+ for folder in os.listdir(ipynb_pipeline_path):
+ log.info("Creating ipynb proxy folders in new Spot locaiton: {0}".format(folder))
+ util.execute_cmd('mkdir -p {0}/ipynb/proxy/{1}/'.format(new_oa_path, folder),log)
+ log.info("Copying advanced mode ipynb template")
+ util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb {0}/ipynb/proxy/{1}/Advanced_Mode.ipynb'.format(new_oa_path, folder),log)
+ log.info("Copying threat investigation ipynb template")
+ util.execute_cmd('cp {0}/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb {0}/ipynb/proxy/{1}/Threat_Investigation.ipynb'.format(new_oa_path, folder),log)
+
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/spot_conf_migration.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/spot_conf_migration.py b/spot-setup/migration/spot_conf_migration.py
new file mode 100755
index 0000000..1ee927b
--- /dev/null
+++ b/spot-setup/migration/spot_conf_migration.py
@@ -0,0 +1,135 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import ConfigParser
+import sys
+import os
+from shutil import copyfile
+from utilities import util
+
+
+def main():
+
+ if len(sys.argv[1:]) < 2:
+ print "Please provide paths to: old_spot.conf , new_spot.conf"
+ sys.exit(1)
+
+ old_conf_file = sys.argv[1]
+ new_conf_file = sys.argv[2]
+
+ log = util.get_logger('SPOT.MIGRATE.CONF')
+
+ old_path = os.path.dirname(os.path.realpath(old_conf_file))
+
+ # create backup for the current configuration file.
+ log.info("Create a backup of /etc/spot.conf before changing it")
+ util.execute_cmd('sudo cp {0} {1}/spot.conf.bkp_0_9'.format(old_conf_file, old_path),log)
+
+ # create configuration objects.
+ old_config = ConfigParser.ConfigParser()
+ current_config = ConfigParser.ConfigParser()
+ new_config = ConfigParser.ConfigParser()
+
+ old_config.readfp(SecHead(open(old_conf_file)))
+ current_config.readfp(SecHead(open(new_conf_file)))
+
+ # create the new conf file.
+ new_config.add_section('conf')
+ for (k,v) in current_config.items("conf"):
+ if old_config.has_option('conf',k):
+ new_config.set('conf',k, old_config.get('conf',k))
+ else:
+ new_config.set('conf',k,v)
+
+ new_path = os.path.dirname(os.path.realpath(new_conf_file))
+ updated_conf_file = '{0}/spot.conf.new'.format(new_path)
+ log.info("Generating merged configuration file in {0}".format(updated_conf_file))
+ formatter(updated_conf_file,new_config)
+
+ log.info("Updating original spot.conf with new and migrated variables and values")
+ util.execute_cmd('sudo cp {0} {1}/spot.conf'.format(updated_conf_file, old_path),log)
+ util.execute_cmd('sudo chmod 0755 {0}/spot.conf'.format(old_path),log)
+
+
+def formatter(conf_file,conf):
+
+ with open(conf_file,'wb') as file:
+
+ file.write("# node configuration\n")
+ file.write("{0}={1}\n".format('UINODE',conf.get('conf','uinode')))
+ file.write("{0}={1}\n".format('MLNODE',conf.get('conf','MLNODE')))
+ file.write("{0}={1}\n".format('GWNODE',conf.get('conf','GWNODE')))
+ file.write("{0}={1}\n".format('DBNAME',conf.get('conf','DBNAME')))
+
+ file.write("# hdfs - base user and data source config\n")
+ file.write("{0}={1}\n".format('HUSER',conf.get('conf','HUSER')))
+ file.write("{0}={1}\n".format('NAME_NODE',conf.get('conf','NAME_NODE')))
+ file.write("{0}={1}\n".format('WEB_PORT',conf.get('conf','WEB_PORT')))
+ file.write("{0}={1}\n".format('DNS_PATH',conf.get('conf','DNS_PATH')))
+ file.write("{0}={1}\n".format('PROXY_PATH',conf.get('conf','PROXY_PATH')))
+ file.write("{0}={1}\n".format('FLOW_PATH',conf.get('conf','FLOW_PATH')))
+ file.write("{0}={1}\n".format('HPATH',conf.get('conf','HPATH')))
+
+ file.write("# impala config\n")
+ file.write("{0}={1}\n".format('IMPALA_DEM',conf.get('conf','IMPALA_DEM')))
+ file.write("{0}={1}\n".format('IMPALA_PORT',conf.get('conf','IMPALA_PORT')))
+
+ file.write("# local fs base user and data source config\n")
+ file.write("{0}={1}\n".format('LUSER',conf.get('conf','LUSER')))
+ file.write("{0}={1}\n".format('LPATH',conf.get('conf','LPATH')))
+ file.write("{0}={1}\n".format('RPATH',conf.get('conf','RPATH')))
+ file.write("{0}={1}\n".format('LIPATH',conf.get('conf','LIPATH')))
+
+ file.write("# suspicious connects config\n")
+ file.write("{0}={1}\n".format('USER_DOMAIN',conf.get('conf','USER_DOMAIN')))
+ file.write("{0}={1}\n".format('SPK_EXEC',conf.get('conf','SPK_EXEC')))
+ file.write("{0}={1}\n".format('SPK_EXEC_MEM',conf.get('conf','SPK_EXEC_MEM')))
+ file.write("{0}={1}\n".format('SPK_DRIVER_MEM',conf.get('conf','SPK_DRIVER_MEM')))
+ file.write("{0}={1}\n".format('SPK_DRIVER_MAX_RESULTS',conf.get('conf','SPK_DRIVER_MAX_RESULTS')))
+ file.write("{0}={1}\n".format('SPK_EXEC_CORES',conf.get('conf','SPK_EXEC_CORES')))
+ file.write("{0}={1}\n".format('SPK_DRIVER_MEM_OVERHEAD',conf.get('conf','SPK_DRIVER_MEM_OVERHEAD')))
+ file.write("{0}={1}\n".format('SPK_EXEC_MEM_OVERHEAD',conf.get('conf','SPK_EXEC_MEM_OVERHEAD')))
+ file.write("{0}={1}\n".format('SPK_AUTO_BRDCST_JOIN_THR',conf.get('conf','SPK_AUTO_BRDCST_JOIN_THR')))
+ file.write("{0}={1}\n".format('LDA_OPTIMIZER',conf.get('conf','LDA_OPTIMIZER')))
+ file.write("{0}={1}\n".format('LDA_ALPHA',conf.get('conf','LDA_ALPHA')))
+ file.write("{0}={1}\n".format('LDA_BETA',conf.get('conf','LDA_BETA')))
+ file.write("{0}={1}\n".format('PRECISION',conf.get('conf','PRECISION')))
+ file.write("{0}={1}\n".format('TOL',conf.get('conf','TOL')))
+ file.write("{0}={1}\n".format('TOPIC_COUNT',conf.get('conf','TOPIC_COUNT')))
+ file.write("{0}={1}\n".format('DUPFACTOR',conf.get('conf','DUPFACTOR')))
+
+class SecHead(object):
+
+ def __init__(self, fp):
+ self.fp = fp
+ self.sechead = '[conf]\n'
+
+ def readline(self):
+ if self.sechead:
+ try:
+ return self.sechead
+ finally:
+ self.sechead = None
+ else:
+ return self.fp.readline()
+
+if __name__ == '__main__':
+ main()
+
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/utilities.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/utilities.py b/spot-setup/migration/utilities.py
new file mode 100644
index 0000000..046679e
--- /dev/null
+++ b/spot-setup/migration/utilities.py
@@ -0,0 +1,89 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+import logging
+
+
+class util(object):
+
+ @classmethod
+ def create_hdfs_folder(cls,hdfs_path,logger):
+ hadoop_create_folder="hadoop fs -mkdir -p {0}".format(hdfs_path)
+ logger.info("SPOT.Utils: Creating hdfs folder: {0}".format(hadoop_create_folder))
+ subprocess.call(hadoop_create_folder,shell=True)
+
+ @classmethod
+ def load_to_hdfs(cls,file_local_path,file_hdfs_path,logger):
+ # move file to hdfs.
+ load_to_hadoop_script = "hadoop fs -moveFromLocal {0} {1}".format(file_local_path,file_hdfs_path)
+ logger.info("SPOT.Utils: Loading file to hdfs: {0}".format(load_to_hadoop_script))
+ subprocess.call(load_to_hadoop_script,shell=True)
+
+ @classmethod
+ def get_logger(cls,logger_name,create_file=False):
+
+ # create logger for prd_ci
+ log = logging.getLogger(logger_name)
+ log.setLevel(level=logging.INFO)
+
+ # create formatter and add it to the handlers
+ formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+
+ if create_file:
+ # create file handler for logger.
+ fh = logging.FileHandler('SPOT.log')
+ fh.setLevel(level=logging.DEBUG)
+ fh.setFormatter(formatter)
+ # reate console handler for logger.
+ ch = logging.StreamHandler()
+ ch.setLevel(level=logging.DEBUG)
+ ch.setFormatter(formatter)
+
+ # add handlers to logger.
+ if create_file:
+ log.addHandler(fh)
+
+ log.addHandler(ch)
+ return log
+
+ @classmethod
+ def execute_cmd(cls,command,logger):
+
+ try:
+ logger.info("SPOT.Utils: Executing: {0}".format(command))
+ subprocess.call(command,shell=True)
+
+ except subprocess.CalledProcessError as e:
+ logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
+ sys.exit(1)
+
+ @classmethod
+ def execute_hive_cmd(cls,command,logger):
+
+ try:
+ logger.info("SPOT.Utils: Executing hive command: hive -e \"{0}\"".format(command))
+ subprocess.call("hive -e \"{0}\"".format(command),shell=True)
+
+ except subprocess.CalledProcessError as e:
+ logger.error("SPOT.Utils: There was an error executing: {0}".format(e.cmd))
+ sys.exit(1)
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/spot.conf
----------------------------------------------------------------------
diff --git a/spot-setup/spot.conf b/spot-setup/spot.conf
index 0209f4e..a0cba3d 100755
--- a/spot-setup/spot.conf
+++ b/spot-setup/spot.conf
@@ -1,3 +1,20 @@
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
#node configuration
UINODE='node03'
MLNODE='node04'
@@ -6,20 +23,16 @@ DBNAME='spot'
#hdfs - base user and data source config
HUSER='/user/spot'
+NAME_NODE=''
+WEB_PORT=50070
DNS_PATH=${HUSER}/${DSOURCE}/hive/y=${YR}/m=${MH}/d=${DY}/
PROXY_PATH=${HUSER}/${DSOURCE}/hive/y=${YR}/m=${MH}/d=${DY}/
FLOW_PATH=${HUSER}/${DSOURCE}/hive/y=${YR}/m=${MH}/d=${DY}/
HPATH=${HUSER}/${DSOURCE}/scored_results/${FDATE}
#impala config
-IMPALA_DEM='node04'
-
-#kerberos config
-KRB_AUTH=false
-KINITPATH=
-KINITOPTS=
-KEYTABPATH=
-KRB_USER=
+IMPALA_DEM=node04
+IMPALA_PORT=21050
#local fs base user and data source config
LUSER='/home/spot'
[15/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala
index f9ee9ce..50a639a 100644
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatPointPrecisionUtility32Test.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.spot.utilities
import org.apache.spark.sql.types._
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala
index 478129a..b6c2b98 100644
--- a/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/FloatingPointUtility64.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.spot.utilities
import org.apache.spark.sql.types._
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/utilities/data/validation/InputSchemaTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/utilities/data/validation/InputSchemaTest.scala b/spot-ml/src/test/scala/org/apache/spot/utilities/data/validation/InputSchemaTest.scala
new file mode 100644
index 0000000..52a3447
--- /dev/null
+++ b/spot-ml/src/test/scala/org/apache/spot/utilities/data/validation/InputSchemaTest.scala
@@ -0,0 +1,53 @@
+package org.apache.spot.utilities.data.validation
+
+import org.apache.spark.sql.types._
+import org.scalatest.{FlatSpec, Matchers}
+
+class InputSchemaTest extends FlatSpec with Matchers {
+
+ "validate" should "return true when incoming schema is valid" in {
+
+ val incomingSchema = StructType(List(StructField("ip", StringType),
+ StructField("ibyt", LongType),
+ StructField("host", StringType),
+ StructField("score", FloatType)))
+
+ val modelSchema = StructType(List(StructField("ip", StringType),
+ StructField("ibyt", LongType),
+ StructField("host", StringType)))
+
+ val results = InputSchema.validate(incomingSchema, modelSchema)
+
+ results.isValid shouldBe true
+ }
+
+ it should "return false when incoming schema is not valid due to type mismatch" in {
+ val incomingSchema = StructType(List(StructField("ip", StringType),
+ StructField("ibyt", StringType),
+ StructField("host", IntegerType),
+ StructField("score", FloatType)))
+
+ val modelSchema = StructType(List(StructField("ip", StringType),
+ StructField("ibyt", LongType),
+ StructField("host", StringType)))
+
+ val results = InputSchema.validate(incomingSchema, modelSchema)
+
+ results.isValid shouldBe false
+ }
+
+ it should "return false when incoming schema is not valid due to required field is missing" in {
+
+ val incomingSchema = StructType(List(StructField("ip", StringType),
+ StructField("ibyt", LongType),
+ StructField("score", FloatType)))
+
+ val modelSchema = StructType(List(StructField("ip", StringType),
+ StructField("ibyt", LongType),
+ StructField("host", StringType)))
+
+ val results = InputSchema.validate(incomingSchema, modelSchema)
+
+ results.isValid shouldBe false
+ }
+}
[08/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb b/spot-oa/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb
index a0a7d26..9860580 100644
--- a/spot-oa/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb
+++ b/spot-oa/oa/flow/ipynb_templates/Threat_Investigation_master.ipynb
@@ -8,6 +8,7 @@
},
"outputs": [],
"source": [
+ "import datetime\n",
"import struct, socket\n",
"import numpy as np\n",
"import linecache, bisect\n",
@@ -16,37 +17,25 @@
"import json\n",
"import os\n",
"import pandas as pd\n",
+ "\n",
"try:\n",
" import ipywidgets as widgets # For jupyter/ipython >= 1.4\n",
"except ImportError:\n",
" from IPython.html import widgets\n",
"from IPython.display import display, Javascript, clear_output\n",
"\n",
- "with open('/etc/spot.conf') as conf:\n",
- " for line in conf.readlines(): \n",
- " if \"DBNAME=\" in line: DBNAME = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
- " elif \"IMPALA_DEM=\" in line: IMPALA_DEM = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
- "\n",
"spath = os.getcwd()\n",
"path = spath.split(\"/\") \n",
"date = path[len(path)-1] \n",
"dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
"cpath = '/'.join(['context' if var == 'ipynb' else var for var in path][:len(path)-2]) + '/'\n",
"\n",
- "sconnect = dpath + 'flow_scores.csv'\n",
- "threats_file = dpath + 'threats.csv'\n",
- "iploc = cpath + 'iploc.csv'\n",
- "nwloc = cpath + 'networkcontext_1.csv'\n",
"anchor = ''\n",
"ir_f = ''\n",
"threat_name = ''\n",
"iplist = ''\n",
"top_results = 20\n",
- "details_limit = 1000\n",
- "if os.path.isfile(iploc):\n",
- " iplist = np.loadtxt(iploc,dtype=np.uint32,delimiter=',',usecols={0}, converters={0: lambda s: np.uint32(s.replace('\"',''))})\n",
- "else:\n",
- " print \"No iploc.csv file was found, Map View map won't be created\""
+ "ip_comments = {}"
]
},
{
@@ -123,7 +112,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
@@ -143,37 +132,68 @@
"def start_investigation(): \n",
" display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\")) \n",
" external_ips = []\n",
- " c_ips=[]\n",
- " clear_output() \n",
+ " clear_output()\n",
+ " global ip_comments\n",
" \n",
- " if os.path.isfile(threats_file) and not file_is_empty(threats_file):\n",
- " with open(threats_file, 'r') as th:\n",
- " t_read = csv.DictReader(th, delimiter='|') \n",
- " for row in t_read: \n",
- " if row['ip'] != '' : c_ips.append(row['ip']) \n",
- "\n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f, delimiter=',') \n",
- " #Internal Netflows use case:\n",
- " for row in reader: \n",
- " if row['sev'] == '1':\n",
+ " response = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " flow{\n",
+ " threats{\n",
+ " list(date:$date) {\n",
+ " srcIp\n",
+ " dstPort\n",
+ " dstIp\n",
+ " srcPort\n",
+ " score \n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " ) \n",
+ " \n",
+ " ip_comments = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " flow{\n",
+ " threats{\n",
+ " comments(date:$date) {\n",
+ " ip\n",
+ " title\n",
+ " text\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " )\n",
+ " ip_comments = ip_comments['data']['flow']['threats']['comments']\n",
+ " if not 'errors' in response : \n",
+ " for row in response['data']['flow']['threats']['list']:\n",
+ " if row['score'] == 1: \n",
" srcIP = ''\n",
" dstIP = '' \n",
- " if row['srcIP'] not in external_ips and row['srcIP'] not in c_ips: \n",
- " external_ips.append(row['srcIP'])\n",
- " if row['dstIP'] not in external_ips and row['dstIP'] not in c_ips: \n",
- " external_ips.append(row['dstIP'])\n",
- "\n",
- " if len(external_ips) == 0:\n",
- " display(widgets.Box((widgets.HTML(value=\"There are no connections scored as High risk.\\\n",
- " You can score some connections at the 'Suspicious' panel.\", width='100%'),)))\n",
- " else: \n",
- " sorted_dict = sorted(external_ips, key=operator.itemgetter(0)) \n",
- " display_controls(sorted_dict) \n",
+ " if row['srcIp'] not in external_ips: \n",
+ " external_ips.append(row['srcIp'])\n",
+ " if row['dstIp'] not in external_ips: \n",
+ " external_ips.append(row['dstIp'])\n",
+ "\n",
+ " if len(external_ips) == 0:\n",
+ " display(widgets.Box((widgets.HTML(value=\"There are no connections scored as High risk.\\\n",
+ " You can score some connections at the 'Suspicious' panel.\", width='100%'),)))\n",
+ " else: \n",
+ " sorted_dict = sorted(external_ips, key=operator.itemgetter(0)) \n",
+ " display_controls(sorted_dict) \n",
+ " else: \n",
" \n",
+ " display(widgets.Box((widgets.HTML(value=\"An error occurred while trying to get the results:\" \n",
+ " + response['errors'][0]['message'], width='100%'),)))\n",
" \n",
"\n",
- "def display_controls(threat_list): \n",
+ "def display_controls(threat_list):\n",
" threat_title.value =\"<h4>Suspicious Connections</h4>\" \n",
" susp_select.options = threat_list\n",
" susp_select.height=150\n",
@@ -186,45 +206,63 @@
" def search_ip(b):\n",
" global anchor \n",
" global top_inbound_b\n",
+ " global expanded_results\n",
" anchor = susp_select.value \n",
" if anchor != \"\":\n",
" clear_output() \n",
" removeWidget(1)\n",
- " print \"Searching for ip: \" + anchor\n",
- " global ir_f \n",
- " ir_f = dpath + \"ir-\" + anchor + \".tsv\"\n",
- "\n",
- " if not os.path.isfile(ir_f) or (os.path.isfile(ir_f) and file_is_empty(ir_f)): \n",
- " imp_query = (\" \\\"SELECT min(treceived) as firstSeen, max(treceived) as lastSeen, sip as srcIP, dip as dstIP, \" + \n",
- " \"sport as SPort, dport AS Dport, count(sip) as conns, max(ipkt) as maxPkts, avg(ipkt) \" + \n",
- " \"as avgPkts, max(ibyt) as maxBytes, avg(ibyt) as avgBytes FROM \"+DBNAME+\".flow WHERE \" + \n",
- " \"y=\"+ date[0:4] +\" AND m=\"+ date[4:6] +\" AND d=\"+ date[6:] +\" \" + \n",
- " \" AND (sip =\\'\" + anchor + \"\\' OR dip=\\'\" + anchor + \"\\') GROUP BY sip, dip,sport,dport\\\" \") \n",
- " !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
- " !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $ir_f\n",
- " clear_output()\n",
" \n",
- " if not file_is_empty(ir_f): \n",
+ " expanded_results = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!,$ip:SpotIpType!){\n",
+ " flow{\n",
+ " threat{\n",
+ " details(date:$date,ip:$ip) {\n",
+ " srcIp\n",
+ " maxBytes\n",
+ " connections\n",
+ " maxPkts\n",
+ " avgPkts\n",
+ " lastSeen\n",
+ " srcPort\n",
+ " firstSeen \n",
+ " dstIp\n",
+ " avgBytes\n",
+ " dstPort\n",
+ " }\n",
+ " }\n",
+ " } \n",
+ " }\n",
+ " \"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'ip': anchor\n",
+ " }\n",
+ " ) \n",
+ " \n",
+ " if not 'errors' in expanded_results : \n",
" print \"\\n Looking for additional details...\"\n",
" display_threat_box(anchor)\n",
"\n",
" get_in_out_and_twoway_conns() \n",
- " add_geospatial_info()\n",
- " add_network_context() \n",
- " \n",
+ " \n",
" display(bottomBox) \n",
- " else:\n",
+ " else: \n",
" display(widgets.Box((widgets.HTML(value=\"Something went wrong. \\\n",
- " The expanded search couldn't be performed\", width='100%'),)))\n",
+ " The expanded search couldn't be performed\" + expanded_results['errors'][0]['message'], width='100%'),)))\n",
" \n",
" search_btn.on_click(search_ip)\n",
"\n",
"\n",
"def display_threat_box(ip):\n",
" clear_output()\n",
+ " global ip_comments \n",
+ " title =\"\" \n",
+ " text = \"\"\n",
+ " title = next((item['title'] for item in ip_comments if item.get(\"ip\") == ip), \"\")\n",
+ " text = next((item['text'] for item in ip_comments if item.get(\"ip\") == ip), \"\") \n",
" result_title.value=\"<h4 class='spot-text-wrapper spot-text-xlg' data-toggle='tooltip'>Threat summary for \" + anchor +\"</h4>\"\n",
- " tc_txt_title = widgets.Text(value='', placeholder='Threat Title', width='100%')\n",
- " tc_txa_summary = widgets.Textarea(value='', height=100, width='95%')\n",
+ " tc_txt_title = widgets.Text(value=title, placeholder='Threat Title', width='100%')\n",
+ " tc_txa_summary = widgets.Textarea(value=text, height=100, width='95%')\n",
" tc_btn_save = widgets.Button(description='Save', width='65px', layout='width:100%')\n",
" tc_btn_save.button_style = 'primary'\n",
" \n",
@@ -241,10 +279,7 @@
" clear_output() \n",
" removeWidget(1) \n",
" response = \"\"\n",
- " response += generate_attack_map_file(anchor, top_inbound_b, top_outbound_b, top_twoway_b)\n",
- " response += generate_stats(anchor, inbound, outbound, twoway, threat_name)\n",
- " response += generate_dendro(anchor, inbound, outbound, twoway, date)\n",
- " response += details_inbound(anchor, top_inbound_b, top_outbound_b, top_twoway_b)\n",
+ " \n",
" response += add_threat(anchor, tc_txt_title.value, tc_txa_summary.value.replace('\\n', '\\\\n'))\n",
" response += \"Story board successfully created for {0}\".format(anchor)\n",
" start_investigation()\n",
@@ -262,234 +297,46 @@
},
"outputs": [],
"source": [
- "def details_inbound(anchor, inbound, outbound, twoway):\n",
- " top_keys = []\n",
- " if len(twoway) > 0: top_keys.extend(twoway.keys())\n",
- " if len(outbound) > 0: top_keys.extend(outbound.keys()) \n",
- " if len(inbound) > 0: top_keys.extend(inbound.keys())\n",
- " sbdet_f = dpath + \"sbdet-\" + anchor + \".tsv\"\n",
- " if not os.path.isfile(sbdet_f):\n",
- " imp_query = (\"\\\"SELECT min(treceived) as tstart, max(treceived) as tend, sip as srcIP, \"\n",
- " + \"dip as dstIP, proto as Proto, sport as SPort, dport AS Dport,ipkt as \"\n",
- " + \"Pkts, ibyt as Bytes FROM \"+DBNAME+\".flow WHERE \"\n",
- " + \"y=\"+ date[0:4] +\" AND m=\"+ date[4:6] +\" AND d=\"+ date[6:]\n",
- " + \" AND ((dip IN({0}) AND sip ='{1}') OR \"\n",
- " + \"(sip IN({0}) \"\n",
- " + \"AND dip ='{1}')) GROUP BY sip, dip, proto, sport, dport, ipkt, ibyt ORDER BY tstart \"\n",
- " + \"LIMIT {2}\\\" \") \n",
- " ips = \"'\" + \"','\".join(top_keys) + \"'\"\n",
- " imp_query = imp_query.format(ips,anchor,details_limit)\n",
- " !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
- " !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $sbdet_f\n",
- "\n",
- " clear_output()\n",
- " return \"Timeline successfully created <br/>\"\n",
- " else:\n",
- " return \"Timeline file already existed <br/>\"\n",
"\n",
- " \n",
- "def generate_dendro(ip, inbound, outbound, twoway, date): \n",
- " dendro_fpath = dpath + 'threat-dendro-' + anchor + \".json\"\n",
- " \n",
- " obj = {\n",
- " 'name':ip,\n",
- " 'children': [],\n",
- " 'time': date \n",
- " }\n",
- " \n",
- " #----- Add Inbound Connections-------#\n",
- " if len(inbound) > 0:\n",
- " obj[\"children\"].append({'name': 'Inbound Only', 'children': [], 'impact': 0}) \n",
- " in_ctxs = {}\n",
- " for ip in inbound:\n",
- " if 'nwloc' in inbound[ip] and len(inbound[ip]['nwloc']) > 0:\n",
- " ctx = inbound[ip]['nwloc'][2] # get the machine type Only for vast Data\n",
- " if ctx not in in_ctxs:\n",
- " in_ctxs[ctx] = 1\n",
- " else:\n",
- " in_ctxs[ctx] += 1\n",
- " for ctx in in_ctxs:\n",
- " obj[\"children\"][0]['children'].append({\n",
- " 'name': ctx,\n",
- " 'impact': in_ctxs[ctx]\n",
- " }) \n",
- " \n",
- " #------ Add Outbound ----------------#\n",
- " if len(outbound) > 0:\n",
- " obj[\"children\"].append({'name': 'Outbound Only', 'children': [], 'impact': 0})\n",
- " out_ctxs = {}\n",
- " for ip in outbound: \n",
- " if 'nwloc' in outbound[ip] and len(outbound[ip]['nwloc']) > 0:\n",
- " ctx = outbound[ip]['nwloc'][2] # get the machine type Only for vast Data\n",
- " if ctx not in out_ctxs:\n",
- " out_ctxs[ctx] = 1\n",
- " else:\n",
- " out_ctxs[ctx] += 1\n",
- " for ctx in out_ctxs:\n",
- " obj[\"children\"][1]['children'].append({\n",
- " 'name': ctx,\n",
- " 'impact': out_ctxs[ctx]\n",
- " }) \n",
- " \n",
- " #------ Add TwoWay ----------------#\n",
- " if len(twoway) > 0:\n",
- " obj[\"children\"].append({'name': 'two way', 'children': [], 'impact': 0})\n",
- " tw_ctxs = {}\n",
- " for ip in twoway:\n",
- " if 'nwloc' in twoway[ip] and len(twoway[ip]['nwloc']) > 0:\n",
- " ctx = twoway[ip]['nwloc'][2] # get the machine type Only for vast Data\n",
- " if ctx not in tw_ctxs:\n",
- " tw_ctxs[ctx] = 1\n",
- " else:\n",
- " tw_ctxs[ctx] += 1\n",
- "\n",
- " for ctx in tw_ctxs:\n",
- " obj[\"children\"][2]['children'].append({\n",
- " 'name': ctx,\n",
- " 'impact': tw_ctxs[ctx]\n",
- " })\n",
- " \n",
- " with open(dendro_fpath, 'w') as dendro_f:\n",
- " dendro_f.write(json.dumps(obj))\n",
- " return \"Incident progression successfully created <br/>\"\n",
- "\n",
- " \n",
- "def generate_stats(ip, inbound, outbound, twoway, threat_name):\n",
- " stats_fpath = dpath + 'stats-' + anchor + \".json\"\n",
- " \n",
- " obj = {\n",
- " 'name':threat_name,\n",
- " 'children': [],\n",
- " 'size': len(inbound) + len(outbound) + len(twoway)\n",
- " }\n",
- " \n",
- " #----- Add Inbound Connections-------#\n",
- " obj[\"children\"].append({'name': 'Inbound Only', 'children': [], 'size': len(inbound)}) \n",
- " in_ctxs = {}\n",
- " for ip in inbound:\n",
- " full_ctx = ''\n",
- " if 'nwloc' in inbound[ip] and len(inbound[ip]['nwloc']) > 0:\n",
- " full_ctx = inbound[ip]['nwloc'][2].split('.')[0]\n",
- " ctx = get_ctx_name(full_ctx) # get the machine type Only for vast Data\n",
- " if ctx not in in_ctxs:\n",
- " in_ctxs[ctx] = 1\n",
- " else:\n",
- " in_ctxs[ctx] += 1\n",
- " for ctx in in_ctxs:\n",
- " obj[\"children\"][0]['children'].append({\n",
- " 'name': ctx,\n",
- " 'size': in_ctxs[ctx]\n",
- " }) \n",
- " \n",
- " \n",
- " #------ Add Outbound ----------------#\n",
- " obj[\"children\"].append({'name': 'Outbound Only', 'children': [], 'size': len(outbound)})\n",
- " out_ctxs = {}\n",
- " for ip in outbound:\n",
- " full_ctx = ''\n",
- " if 'nwloc' in outbound[ip] and len(outbound[ip]['nwloc']) > 0:\n",
- " full_ctx = outbound[ip]['nwloc'][2].split('.')[0]\n",
- " ctx = get_ctx_name(full_ctx) # get the machine type Only for vast Data\n",
- " if ctx not in out_ctxs:\n",
- " out_ctxs[ctx] = 1\n",
- " else:\n",
- " out_ctxs[ctx] += 1\n",
- " for ctx in out_ctxs:\n",
- " obj[\"children\"][1]['children'].append({\n",
- " 'name': ctx,\n",
- " 'size': out_ctxs[ctx]\n",
- " }) \n",
- " \n",
- " #------ Add Twoway ----------------#\n",
- " obj[\"children\"].append({'name': 'two way', 'children': [], 'size': len(twoway)})\n",
- " tw_ctxs = {}\n",
- " for ip in twoway:\n",
- " full_ctx = ''\n",
- " if 'nwloc' in twoway[ip] and len(twoway[ip]['nwloc']) > 0:\n",
- " full_ctx = twoway[ip]['nwloc'][2].split('.')[0]\n",
- " ctx = get_ctx_name(full_ctx) # get the machine type Only for vast Data\n",
- " if ctx not in tw_ctxs:\n",
- " tw_ctxs[ctx] = 1\n",
- " else:\n",
- " tw_ctxs[ctx] += 1\n",
- " \n",
- " for ctx in tw_ctxs:\n",
- " obj[\"children\"][2]['children'].append({\n",
- " 'name': ctx,\n",
- " 'size': tw_ctxs[ctx]\n",
- " })\n",
- " \n",
- " json_str = json.dumps(obj)\n",
- " with open(stats_fpath, 'w') as stats_f:\n",
- " stats_f.write(json_str)\n",
- " return \"Stats file successfully created <br/>\"\n",
- "\n",
- " \n",
- "def get_ctx_name(full_context): \n",
- " ctx= 'DMZ'\n",
- " if \"VPN\" in full_context:\n",
- " ctx = \"VPN\" \n",
- " elif \"DMZ\" in full_context:\n",
- " ctx = \"DMZ\"\n",
- " elif \"Proxy\" in full_context:\n",
- " ctx = \"Proxy\" \n",
- " elif \"FW\" in full_context:\n",
- " ctx = \"FW\" \n",
- " return ctx\n",
- "\n",
- "\n",
- "# calculate number of inbound only, two-way, and outbound only\n",
- "# build dict of IP addresses\n",
- "# firstSeen,lastSeen,srcIP, dstIP, sport,dport,conns, maxPkts, avgPkts,maxBytes, avgBytes\n",
"def get_in_out_and_twoway_conns():\n",
- " global inbound\n",
- " inbound = {}\n",
- " global outbound\n",
- " outbound = {}\n",
- " global twoway\n",
- " twoway = {}\n",
" srcdict = {}\n",
" dstdict = {}\n",
" conns_dict= {} \n",
- " rowct = 0\n",
- " if os.path.isfile(ir_f): \n",
- " df = pd.read_csv(ir_f,sep='\\t') \n",
- " with open(ir_f, 'r') as f:\n",
- " reader = csv.reader(f,delimiter='\\t')\n",
- " reader.next() #skip headers \n",
- " for row in reader: \n",
- " if row != []:\n",
- " srcdict[row[2]] = {\n",
- " 'ip_int': struct.unpack(\"!L\", socket.inet_aton(row[2]))[0],\n",
- " 'dst_ip': row[3],\n",
- " 'dst_ip_int': struct.unpack(\"!L\", socket.inet_aton(row[3]))[0],\n",
- " 'conns': int(row[6]),\n",
- " 'maxbytes': int(row[9])\n",
- " }\n",
- " dstdict[row[3]] = {\n",
- " 'ip_int': struct.unpack(\"!L\", socket.inet_aton(row[3]))[0],\n",
- " 'src_ip': row[2],\n",
- " 'src_ip_int': struct.unpack(\"!L\", socket.inet_aton(row[2]))[0],\n",
- " 'conns': int(row[6]),\n",
- " 'maxbytes': int(row[9])\n",
- " } \n",
- " rowct +=1 \n",
+ " rowct = 0 \n",
+ " if not 'errors' in expanded_results : \n",
+ " df = pd.DataFrame(expanded_results['data']['flow']['threat']['details']) \n",
+ " for row in expanded_results['data']['flow']['threat']['details']: \n",
+ " srcdict[row['srcIp']] = {\n",
+ " 'ip_int': struct.unpack(\"!L\", socket.inet_aton(str(row['srcIp'])))[0],\n",
+ " 'dst_ip': row['dstIp'],\n",
+ " 'dst_ip_int': struct.unpack(\"!L\", socket.inet_aton(str(row['dstIp'])))[0],\n",
+ " 'conns': int(row['connections']),\n",
+ " 'maxbytes': int(row['maxBytes'])\n",
+ " }\n",
+ " dstdict[row['dstIp']] = {\n",
+ " 'ip_int': struct.unpack(\"!L\", socket.inet_aton(str(row['dstIp'])))[0],\n",
+ " 'src_ip': row['srcIp'],\n",
+ " 'src_ip_int': struct.unpack(\"!L\", socket.inet_aton(str(row['srcIp'])))[0],\n",
+ " 'conns': int(row['connections']),\n",
+ " 'maxbytes': int(row['maxBytes'])\n",
+ " } \n",
+ " rowct +=1 \n",
" \n",
- " src = df.loc[df['dstip'] == anchor]\n",
- " src_per_conns = src.sort_values('conns',0,False)\n",
- " src_per_bytes = src.sort_values('maxbytes',0,False)\n",
- " dst = df.loc[df['srcip'] == anchor]\n",
- " dst_per_conns = dst.sort_values('conns',0,False)\n",
- " dst_per_bytes = dst.sort_values('maxbytes',0,False)\n",
+ " src = df.loc[df['dstIp'] == anchor]\n",
+ " src_per_conns = src.sort_values('connections',0,False)\n",
+ " src_per_bytes = src.sort_values('maxBytes',0,False)\n",
+ " dst = df.loc[df['srcIp'] == anchor]\n",
+ " dst_per_conns = dst.sort_values('connections',0,False)\n",
+ " dst_per_bytes = dst.sort_values('maxBytes',0,False)\n",
"\n",
" children = []\n",
- " children += (display_results(['srcip','conns','sport','dport'], src_per_conns, \n",
+ " children += (display_results(['srcIp','connections','srcPort','dstPort'], src_per_conns, \n",
" top_results),)\n",
- " children += (display_results(['dstip','conns','sport','dport'], dst_per_conns, \n",
+ " children += (display_results(['dstIp','connections','srcPort','dstPort'], dst_per_conns, \n",
" top_results),)\n",
- " children += (display_results(['srcip','maxbytes','sport','dport'], src_per_bytes, \n",
+ " children += (display_results(['srcIp','maxBytes','srcPort','dstPort'], src_per_bytes, \n",
" top_results),)\n",
- " children += (display_results(['dstip','maxbytes','sport','dport'], dst_per_bytes, \n",
+ " children += (display_results(['dstIp','maxBytes','srcPort','dstPort'], dst_per_bytes, \n",
" top_results),)\n",
"\n",
" result_tabs = widgets.Accordion(children=children, width='100%', selected_index=-1)\n",
@@ -505,245 +352,51 @@
"\n",
" resultTableBox.children = [result_tabs,]\n",
" \n",
- " if rowct > 0:\n",
- " for result in srcdict:\n",
- " if result in dstdict:\n",
- " twoway[result] = srcdict[result]\n",
- " else:\n",
- " outbound[result] = srcdict[result]\n",
- "\n",
- " for result in dstdict:\n",
- " if result not in srcdict:\n",
- " inbound[result] = dstdict[result] \n",
- " \n",
- " global top_inbound_b\n",
- " global top_outbound_b\n",
- " global top_twoway_b\n",
- " if len(inbound) > 0:\n",
- " top_inbound_b = get_top_bytes(inbound,top_results)\n",
- " top_inbound_conns = get_top_conns(inbound,top_results)\n",
- " top_inbound_b.update(top_inbound_conns) # merge the two dictionaries\n",
- " if len(outbound) > 0:\n",
- " top_outbound_b = get_top_bytes(outbound,top_results)\n",
- " top_outbound_conns = get_top_conns(outbound,top_results)\n",
- " top_outbound_b.update(top_outbound_conns) # merge the two dictionaries\n",
- " if len(twoway) > 0: \n",
- " top_twoway_b = get_top_bytes(twoway,top_results)\n",
- " top_twoway_conns = get_top_conns(twoway,top_results)\n",
- " top_twoway_b.update(top_twoway_conns) # merge the two dictionaries\n",
"\n",
"def display_results(cols, dataframe, top): \n",
" table = dataframe[:top].to_html(classes='table table-striped table-bordered table-hover', columns=cols, index=False)\n",
" return widgets.HTML(value=table, width='100%')\n",
- " \n",
- "#=========== Adds GEO IP information to the outbound, inbound and twoway connections==============================# \n",
- "def add_geospatial_info():\n",
- " # get geospatial info, only when iplocation file is available\n",
- " if iplist != '':\n",
- " for srcip in outbound:\n",
- " reader = csv.reader([linecache.getline(iploc, bisect.bisect(iplist,outbound[srcip]['ip_int'])).replace('\\n','')])\n",
- " outbound[srcip]['geo'] = reader.next()\n",
- " reader = csv.reader([linecache.getline(iploc, bisect.bisect(iplist,outbound[srcip]['dst_ip_int'])).replace('\\n','')])\n",
- " outbound[srcip]['geo_dst'] = reader.next()\n",
- "\n",
- " for dstip in twoway:\n",
- " reader = csv.reader([linecache.getline(iploc, bisect.bisect(iplist,twoway[dstip]['ip_int'])).replace('\\n','')])\n",
- " twoway[dstip]['geo'] = reader.next()\n",
- "\n",
- " for srcip in inbound:\n",
- " reader = csv.reader([linecache.getline(iploc, bisect.bisect(iplist,inbound[srcip]['ip_int'])).replace('\\n','')])\n",
- " inbound[srcip]['geo'] = reader.next()\n",
- " reader = csv.reader([linecache.getline(iploc, bisect.bisect(iplist,inbound[srcip]['src_ip_int'])).replace('\\n','')])\n",
- " inbound[srcip]['geo_src'] = reader.next()\n",
- " \n",
- " \n",
- " \n",
- "# need some way to combine timelines of outbound and two-way with big picture inbound only\n",
- "# get network context - get start and end ranges\n",
- "def add_network_context():\n",
- " nwdict = {}\n",
- " if os.path.isfile(nwloc) : \n",
- " with open(nwloc, 'r') as f:\n",
- " reader = csv.reader(f,delimiter=',')\n",
- " reader.next()\n",
- " #address range, description\n",
- " for row in reader:\n",
- "\n",
- " if '/' in row[0]: \n",
- " #Range in subnet\n",
- " iprange = row[0].split('/')\n",
- " if len(iprange) < 2:\n",
- " ipend = 0\n",
- " else:\n",
- " ipend = int(iprange[1])\n",
- " nwdict[row[0]] = [struct.unpack(\"!L\", socket.inet_aton(iprange[0]))[0],\n",
- " struct.unpack(\"!L\", socket.inet_aton(iprange[0]))[0]+2**(32-ipend)-1, row[1]] \n",
- " elif '-' in row[0]: \n",
- " #IP Range \n",
- " iprange = row[0].split('-') \n",
- " nwdict[row[0]] = [struct.unpack(\"!L\", socket.inet_aton(iprange[0].replace(\" \", \"\")))[0],\n",
- " struct.unpack(\"!L\", socket.inet_aton(iprange[1].replace(\" \", \"\")))[0], row[1]]\n",
- " else:\n",
- " #Exact match \n",
- " nwdict[row[0]] = [struct.unpack(\"!L\", socket.inet_aton(row[0]))[0],\n",
- " struct.unpack(\"!L\", socket.inet_aton(row[0]))[0], row[1]] \n",
- "\n",
- " for srcip in outbound: \n",
- " temp_ip = struct.unpack(\"!L\", socket.inet_aton(srcip))[0]\n",
- " if srcip in nwdict:\n",
- " inbound[srcip]['nwloc'] = nwdict[srcip]\n",
- " else:\n",
- " matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]\n",
- " outbound[srcip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else '' \n",
- "\n",
- " for dstip in twoway: \n",
- " temp_ip = struct.unpack(\"!L\", socket.inet_aton(dstip))[0]\n",
- " if dstip in nwdict:\n",
- " twoway[dstip]['nwloc'] = nwdict[dstip]\n",
- " else:\n",
- " matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]\n",
- " twoway[dstip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''\n",
- "\n",
- " for srcip in inbound:\n",
- " temp_ip = struct.unpack(\"!L\", socket.inet_aton(srcip))[0]\n",
- " if srcip in nwdict:\n",
- " inbound[srcip]['nwloc'] = nwdict[srcip]\n",
- " else:\n",
- " matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]\n",
- " inbound[srcip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''\n",
- "\n",
- " \n",
- "def generate_attack_map_file(ip, inbound, outbound, twoway): \n",
- " if iplist != '':\n",
- " globe_fpath = dpath + 'globe-' + ip + \".json\"\n",
- " globe_json = {}\n",
- " globe_json['type'] = \"FeatureCollection\"\n",
- " globe_json['sourceips'] = []\n",
- " globe_json['destips'] = [] \n",
- " for srcip in twoway:\n",
- " try:\n",
- " row = twoway[srcip]['geo'] \n",
- " globe_json['destips'].append({\n",
- " 'type': 'Feature',\n",
- " 'properties': {\n",
- " 'location':row[8],\n",
- " 'ip':srcip,\n",
- " 'type':1\n",
- " },\n",
- " 'geometry': {\n",
- " 'type': 'Point',\n",
- " 'coordinates': [float(row[7]), float(row[6])]\n",
- " }\n",
- " })\n",
- " except ValueError:\n",
- " pass\n",
- "\n",
- "\n",
- " for dstip in outbound:\n",
- " try:\n",
- " row = outbound[dstip]['geo']\n",
- " dst_geo = outbound[dstip]['geo_dst']\n",
- " globe_json['sourceips'].append({\n",
- " 'type': 'Feature',\n",
- " 'properties': {\n",
- " 'location':row[8],\n",
- " 'ip':dstip,\n",
- " 'type':3\n",
- " },\n",
- " 'geometry': {\n",
- " 'type': 'Point',\n",
- " 'coordinates': [float(row[7]), float(row[6])]\n",
- " }\n",
- " })\n",
- " globe_json['destips'].append({\n",
- " 'type': 'Feature',\n",
- " 'properties': {\n",
- " 'location':row[8],\n",
- " 'ip':outbound[dstip]['dst_ip'],\n",
- " 'type':3\n",
- " },\n",
- " 'geometry': {\n",
- " 'type': 'Point',\n",
- " 'coordinates': [float(dst_geo[7]), float(dst_geo[6])]\n",
- " }\n",
- " }) \n",
- "\n",
- " except ValueError:\n",
- " pass\n",
"\n",
- " for dstip in inbound:\n",
- " try:\n",
- " row = inbound[dstip]['geo']\n",
- " dst_geo = inbound[dstip]['geo_src']\n",
- " globe_json['sourceips'].append({\n",
- " 'type': 'Feature',\n",
- " 'properties': {\n",
- " 'location':row[8],\n",
- " 'ip':dstip,\n",
- " 'type':2\n",
- " },\n",
- " 'geometry': {\n",
- " 'type': 'Point',\n",
- " 'coordinates': [float(row[7]), float(row[6])]\n",
- " }\n",
- " })\n",
- " globe_json['destips'].append({\n",
- " 'type': 'Feature',\n",
- " 'properties': {\n",
- " 'location':row[8],\n",
- " 'ip':inbound[dstip]['src_ip'],\n",
- " 'type':2\n",
- " },\n",
- " 'geometry': {\n",
- " 'type': 'Point',\n",
- " 'coordinates': [float(dst_geo[7]), float(dst_geo[6])]\n",
- " }\n",
- " })\n",
- " except ValueError:\n",
- " pass\n",
- "\n",
- " json_str = json.dumps(globe_json)\n",
- " with open(globe_fpath, 'w') as globe_f:\n",
- " globe_f.write(json_str)\n",
- " response = \"Geolocation map successfully created <br/>\"\n",
- " else:\n",
- " response = \"The map can't be created without an iploc file <br/>\" \n",
- " \n",
- " return response\n",
"\n",
" \n",
"def add_threat(ip,threat_title, threat_comment):\n",
- " content = ''\n",
- " try:\n",
- " threat_f = open(threats_file, 'r')\n",
- " content = threat_f.read()\n",
- " if '{0}|{1}|{2}\\n'.format(ip,threat_title,threat_comment) not in content:\n",
- " content += '{0}|{1}|{2}\\n'.format(ip,threat_title,threat_comment)\n",
- " threat_f.close() \n",
- " except:\n",
- " content = 'ip|title|summary\\n'\n",
- " content += '{0}|{1}|{2}\\n'.format(ip,threat_title,threat_comment)\n",
- " \n",
- " threat_fw = open(threats_file, 'w')\n",
- " threat_fw.write(content)\n",
- " threat_fw.close()\n",
- " return \"\"\n",
- "\n",
+ " \n",
+ " mutation=\"\"\"mutation(\n",
+ " $date: SpotDateType, \n",
+ " $ip: SpotIpType!, \n",
+ " $text: String!, \n",
+ " $title: String!,\n",
+ " $threatDetails: [NetflowThreatDetailsInputType]!,\n",
+ " $topResults:Int) \n",
+ " {\n",
+ " flow {\n",
+ " createStoryboard(input:{\n",
+ " threatDetails: $threatDetails,\n",
+ " date: $date, \n",
+ " ip: $ip, \n",
+ " title: $title, \n",
+ " text: $text,\n",
+ " first:$topResults})\n",
+ " {success}\n",
+ " }\n",
+ " }\"\"\"\n",
+ " \n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'ip': ip,\n",
+ " 'title': threat_title,\n",
+ " 'text': threat_comment,\n",
+ " 'threatDetails': expanded_results['data']['flow']['threat']['details'],\n",
+ " 'first':top_results\n",
+ "\n",
+ " }\n",
" \n",
- "def get_top_bytes(conns_dict, top):\n",
- " topbytes = sorted(conns_dict.iteritems(), key=lambda (x,y): y['maxbytes'], reverse=True)\n",
- " topbytes = topbytes[0:top]\n",
- " return dict(topbytes)\n",
- "\n",
- "\n",
- "def get_top_conns(conns_dict, top): \n",
- " topconns = sorted(conns_dict.iteritems(), key=lambda (x,y): y['conns'], reverse=True)\n",
- " topconns = topconns[0:top]\n",
- " return dict(topconns)\n",
+ " response = GraphQLClient.request(mutation,variables)\n",
+ " if not 'errors' in response:\n",
+ " return \"Story board successfully created\"\n",
+ " else:\n",
+ " return response['errors'][0]['message']\n",
" \n",
- "def file_is_empty(path):\n",
- " return os.stat(path).st_size==0\n",
- "\n",
"def removeWidget(index):\n",
" js_command = \"$('.widget-area > .widget-subarea > .widget-box:eq({0})').remove();\".format(index) \n",
" display(Javascript(js_command))"
@@ -754,8 +407,8 @@
"execution_count": null,
"metadata": {
"collapsed": false
- },
- "outputs": [],
+ },
+ "outputs": [],
"source": [
"start_investigation()"
]
@@ -777,7 +430,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython2",
- "version": "2.7.10"
+ "version": "2.7.5"
}
},
"nbformat": 4,
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/README.md b/spot-oa/oa/proxy/README.md
index ac6f73c..70c8b00 100644
--- a/spot-oa/oa/proxy/README.md
+++ b/spot-oa/oa/proxy/README.md
@@ -1,10 +1,10 @@
# PROXY
-Proxy sub-module will extract and transform Proxy data already ranked by spot-ml and will load into csv files for presentation layer.
+Proxy sub-module will extract and transform Proxy data already ranked by spot-ml and will load it into impala tables for the presentation layer.
## Proxy Components
-###proxy_oa.py
+### proxy_oa.py
Proxy spot-oa main script executes the following steps:
@@ -13,7 +13,6 @@ Proxy spot-oa main script executes the following steps:
data: data/proxy/<date>/
ipython Notebooks: ipynb/proxy/<date>/
-
2. Creates a copy of the notebooks templates into the ipython Notebooks path and renames them removing the "_master" part from the name.
3. Gets the proxy_results.csv from the HDFS location according to the selected date, and copies it back to the corresponding data path.
@@ -30,11 +29,9 @@ Proxy spot-oa main script executes the following steps:
9. Creates a hash for every full_uri + clientip pair to use as filename.
- 10. Saves proxy_scores.tsv file.
-
- 11. Creates a backup of proxy_scores.tsv file.
+ 10. Saves the data in the _proxy_\scores_ table.
- 12. Creates proxy data details files.
+ 12. Collects information about additional connections to display the details table in the UI.
**Dependencies**
@@ -60,63 +57,70 @@ Before running Proxy OA, users need to configure components for the first time.
**Output**
-- proxy_scores.tsv: Main results file for Proxy OA. This file is tab separated and it's limited to the number of rows the user selected when running [oa/start_oa.py](/spot-oa/oa/INSTALL.md#usage).
-
- Schema with zero-indexed columns:
-
- 0.p_date: string
- 1.p_time: string
- 2.clientip: string
- 3.host: string
- 4.reqmethod: string
- 5.useragent: string
- 6.resconttype: string
- 7.duration: int
- 8.username: string
- 9.webcat: string
- 10.referer: string
- 11.respcode: string
- 12.uriport: string
- 13.uripath: string
- 14.uriquery: string
- 15.serverip: string
- 16.scbytes: int
- 17.csbytes: int
- 18.fulluri: string
- 19.word: string
- 20.score: string
- 21.uri_rep: string
- 22.uri_sev: string
- 23.respcode_name: string
- 24.network_context: string
- 25.hash: string
-
-
-- proxy_scores_bu.tsv: The backup file of suspicious connects in case user want to roll back any changes made during analysis. Schema is same as proxy_scores.tsv.
-
-
-- edge-clientip-\<hash>HH.tsv: One file for each fulluri + clientip connection for each hour of the day.
-
- Schema with zero-indexed columns:
-
- 0.p_date: string
- 1.p_time: string
- 2.clientip: string
- 3.host: string
- 4.webcat: string
- 5.respcode: string
- 6.reqmethod: string
- 7.useragent: string
- 8.resconttype: string
- 9.referer: string
- 10.uriport: string
- 11.serverip: string
- 12.scbytes: int
- 13.csbytes: int
- 14.fulluri: string
-
-###proxy_conf.json
-This file is part of the initial configuration for the proxy pipeline It will contain mapped all the columns included in the proxy_results.csv and proxy_scores.tsv files.
+- Proxy suspicious connections. _proxy\_scores_ table.
+
+Main results file for Proxy OA. The data stored in this table is limited by the number of rows the user selected when running [oa/start_oa.py](/spot-oa/oa/INSTALL.md#usage).
+
+ 0.tdate string
+ 1.time string
+ 2.clientip string
+ 3.host string
+ 4.reqmethod string
+ 5.useragent string
+ 6.resconttype string
+ 7.duration int
+ 8.username string
+ 9.webcat string
+ 10.referer string
+ 11.respcode string
+ 12.uriport string
+ 13.uripath string
+ 14.uriquery string
+ 15.serverip string
+ 16.scbytes int
+ 17.csbytes int
+ 18.fulluri string
+ 19.word string
+ 20.ml_score Float
+ 21.uri_rep string
+ 22.respcode_name string
+ 23.network_context string
+
+
+- Proxy details. _proxy\_edge_ table.
+
+A query will be executed for each fulluri + clientip connection for each hour of the day.
+
+ 0.tdate STRING
+ 1.time STRING
+ 2.clientIp STRING
+ 3.host string
+ 4.webcat string
+ 5.respcode string
+ 6.reqmethod string
+ 7.useragent string
+ 8.resconttype string
+ 9.referer string
+ 10.uriport string
+ 11.serverip string
+ 12.scbytes int
+ 13.csbytes int
+ 14.fulluri string
+ 15.hh int
+ 16.respcode_name string
+
+
+- Proxy Ingest summary. _proxy\_ingest\_summary_ table.
+
+This table is populated with the number of connections ingested by minute during that day.
+
+ Table schema:
+ 0. tdate: string
+ 1. total: bigint
+
+
+### proxy_conf.json
+This file is part of the initial configuration for the proxy pipeline It will contain mapped all the columns included in the proxy_results.csv and proxy tables.
This file contains three main arrays:
@@ -130,6 +134,9 @@ After OA process completes, a copy of each iPython notebook is going to be copie
With these iPython notebooks user will be able to perform further analysis and score connections. User can also
experiment adding or modifying the code.
If a new functionality is required for the ipython notebook, the templates need to be modified to include the functionality for new executions.
-For further reference on how to work with these notebooks, you can read:
-[Edge Notebook.ipynb](/spot-oa/oa/proxy/ipynb_templates/EdgeNotebook.md)
+For further reference on how to work with these notebooks, you can read:
[Threat_Investigation.ipynb](/spot-oa/oa/proxy/ipynb_templates/ThreatInvestigation.md)
+
+
+### Reset scored connections
+To reset all scored connections for a day, a specific cell with a preloaded function is included in the Advanced Mode Notebook. The cell is commented to avoid accidental executions, but is properly labeled.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/__init__.py b/spot-oa/oa/proxy/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/proxy/__init__.py
+++ b/spot-oa/oa/proxy/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb b/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb
new file mode 100644
index 0000000..f9d9583
--- /dev/null
+++ b/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb
@@ -0,0 +1,296 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Apache Spot's Ipython Advanced Mode\n",
+ "## Proxy\n",
+ "\n",
+ "This guide provides examples about how to request data, show data with some cool libraries like pandas and more.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Import Libraries**\n",
+ "\n",
+ "The next cell will import the necessary libraries to execute the functions. Do not remove"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import linecache, bisect\n",
+ "import os\n",
+ "\n",
+ "spath = os.getcwd()\n",
+ "path = spath.split(\"/\")\n",
+ "date = path[len(path)-1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Request Data**\n",
+ "\n",
+ "In order to request data we are using Graphql (a query language for APIs, more info at: http://graphql.org/).\n",
+ "\n",
+ "We provide the function to make a data request, all you need is a query and variables\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "def makeGraphqlRequest(query, variables):\n",
+ " return GraphQLClient.request(query, variables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now that we have a function, we can run a query like this:\n",
+ "\n",
+ "*Note: There's no need to manually set the date for the query, by default the code will read the date from the current path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "suspicious_query = \"\"\"query($date:SpotDateType) {\n",
+ " proxy {\n",
+ " suspicious(date:$date)\n",
+ " { clientIp\n",
+ " clientToServerBytes\n",
+ " datetime\n",
+ " duration\n",
+ " host\n",
+ " networkContext\n",
+ " referer\n",
+ " requestMethod\n",
+ " responseCode\n",
+ " responseCodeLabel\n",
+ " responseContentType\n",
+ " score\n",
+ " serverIp\n",
+ " serverToClientBytes\n",
+ " uri\n",
+ " uriPath\n",
+ " uriPort\n",
+ " uriQuery\n",
+ " uriRep\n",
+ " userAgent\n",
+ " username\n",
+ " webCategory \n",
+ " }\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ "##If you want to use a different date for your query, switch the \n",
+ "##commented/uncommented following lines\n",
+ "\n",
+ "variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# 'date': \"2016-10-08\"\n",
+ " }\n",
+ " \n",
+ "suspicious_request = makeGraphqlRequest(suspicious_query,variables)\n",
+ "\n",
+ "##The variable suspicious_request will contain the resulting data from the query.\n",
+ "results = suspicious_request['data']['proxy']['suspicious']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Pandas Dataframes\n",
+ "\n",
+ "The following cell loads the results into a pandas dataframe\n",
+ "\n",
+ "For more information on how to use pandas, you can learn more here: https://pandas.pydata.org/pandas-docs/stable/10min.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "df = pd.read_json(json.dumps(results))\n",
+ "##Printing only the selected column list from the dataframe\n",
+ "##Unless specified otherwise, \n",
+ "print df[['clientIp','uriQuery','datetime','clientToServerBytes','serverToClientBytes', 'host']]\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Additional operations \n",
+ "\n",
+ "Additional operations can be performed on the dataframe like sorting the data, filtering it and grouping it\n",
+ "\n",
+ "**Filtering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "##Filter results where the destination port = 3389\n",
+ "##The resulting data will be stored in df2 \n",
+ "\n",
+ "df2 = df[df['clientIp'].isin(['10.173.202.136'])]\n",
+ "print df2[['clientIp','uriQuery','datetime','host']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Ordering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "srtd = df.sort_values(by=\"host\")\n",
+ "print srtd[['host','clientIp','uriQuery','datetime']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Grouping the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "## This command will group the results by pairs of source-destination IP\n",
+ "## summarizing all other columns \n",
+ "grpd = df.groupby(['clientIp','host']).sum()\n",
+ "## This will print the resulting dataframe displaying the input and output bytes columnns\n",
+ "print grpd[[\"clientToServerBytes\",\"serverToClientBytes\"]]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Reset Scored Connections**\n",
+ "\n",
+ "Uncomment and execute the following cell to reset all scored connections for this day"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# reset_scores = \"\"\"mutation($date:SpotDateType!) {\n",
+ "# proxy{\n",
+ "# resetScoredConnections(date:$date){\n",
+ "# success\n",
+ "# }\n",
+ "# }\n",
+ "# }\"\"\"\n",
+ "\n",
+ "\n",
+ "# variables={\n",
+ "# 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# }\n",
+ " \n",
+ "# request = makeGraphqlRequest(reset_scores,variables)\n",
+ "\n",
+ "\n",
+ "# print request['data']['proxy']['resetScoredConnections']['success']"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Sandbox\n",
+ "\n",
+ "At this point you can perform your own analysis using the previously provided functions as a guide.\n",
+ "\n",
+ "Happy threat hunting!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "#Your code here"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/ipynb_templates/EdgeNotebook.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/EdgeNotebook.md b/spot-oa/oa/proxy/ipynb_templates/EdgeNotebook.md
deleted file mode 100644
index 2f9472c..0000000
--- a/spot-oa/oa/proxy/ipynb_templates/EdgeNotebook.md
+++ /dev/null
@@ -1,74 +0,0 @@
-#PROXY Edge Investigation Notebook
-
-###Dependencies
-- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html)
-- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/)
-- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip)
-
-The following python modules will be imported for the notebook to work correctly:
-
- import urllib2
- import json
- import os
- import csv
- import datetime
- import subprocess
- import ipywidgets #For jupyter/ipython >= 1.4
- from IPython.html import widgets # For jupyter/ipython < 1.4
- from IPython.display import display, HTML, clear_output, Javascript
-
-
-###Pre-requisites
-- Execution of the spot-oa process for Proxy
-- Correct setup the spot.conf file [Read more](/wiki/Edit%20Solution%20Configuration)
-- Have a public key created between the current UI node and the ML node. [Read more](/wiki/Configure%20User%20Accounts#configure-user-accounts)
-
-
-###Data
-The whole process in this notebook depends entirely on the existence of `proxy_scores.tsv` file, which is generated at the OA process.
-The data is directly manipulated on the .tsv files, so a `proxy_scores_bu.tsv` is created as a backup to allow the user to restore the original data at any point,
-and this can be performed executing the last cell on the notebook with the following command.
-
- !cp $sconnectbu $sconnect
-
-
-**Input files**
-All these paths should be relative to the main OA path.
-Schema for these files can be found [here](/spot-oa/oa/proxy)
-
- data/proxy/<date>/proxy_scores.tsv
- data/proxy/<date>/proxy_scores_bu.tsv
-
-**Temporary Files**
-
- data/proxy/<date>/proxy_scores_tmp.tsv
-
-**Output files**
-
- data/proxy/<date>/proxy_scores.tsv (Updated with severity values)
- data/proxy/<date>/proxy_scores_fb.csv (File with scored connections that will be used for ML feedback)
-
-
-###Functions
-**Widget configuration**
-This is not a function, but more like global code to set up styles and widgets to format the output of the notebook.
-
-`data_loader():` - This function loads the source file into a csv dictionary reader to create a list with all disctinct full_uri values.
-
-`fill_list(list_control,source):` - This function loads the given dictionary into a listbox and appends an empty item at the top with the value '--Select--' (Just for design sake)
-
-` assign_score(b):` - This event is executed when the user clicks the 'Score' button.
-If the 'Quick scoring' textbox is not empty, the notebook will read that value and ignore any selection made in the listbox, otherwise the sought value will be obtained from the listbox.
-A linear search will be performed in the `proxy_scores.tsv` file to find all `full_uri` values matching the sought .
-In every matching row found, the `uri_sev` value will be updated according to the 'Rating' value selected in the radio button list.
-All of the rows will then be appended to the `proxy_scores_tmp.tsv` file. At the end of this process, this file will replace the original `proxy_scores.tsv`.
-
-Only the scored rows will also be appended to the `proxy_scores_fb.csv` file, which will later be used for the ML feedback.
-
-`save(b):` -This event is triggered by the 'Save' button, first it will remove the widget area and call the `load_data()` function to start the loading process again, this will
-refresh the listbox removing all scored URIs.
-A javascript function is also executed to refresh the other panels in the suspicious connects page removing the need of a manual refresh.
-Afterwards the `ml_feedback()` function will be invoqued.
-
-`ml_feedback():` - A shell script is executed, transferring thru secure copy the _proxy_scores_fb.csv_ file into ML Master node, where the destination path is defined at the spot.conf file.
-
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb b/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb
deleted file mode 100644
index 0cfc28b..0000000
--- a/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb
+++ /dev/null
@@ -1,229 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Get Suspicious Requests"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import urllib2\n",
- "import json\n",
- "import os\n",
- "import csv \n",
- "\n",
- "# getting date from the parent path. \n",
- "path = os.getcwd().split(\"/\") \n",
- "date = path[len(path)-1] \n",
- "dsource = path[len(path)-2] \n",
- "dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
- "\n",
- "sconnect = dpath + 'proxy_scores.tsv'\n",
- "sconnectbu = dpath + 'proxy_scores_bu.tsv'\n",
- "score_tmp = dpath + 'proxy_tmp.tsv' \n",
- "score_fbk = dpath + 'proxy_scores_fb.csv'"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "try:\n",
- " import ipywidgets as widgets # For jupyter/ipython >= 1.4\n",
- "except ImportError:\n",
- " from IPython.html import widgets\n",
- "from IPython.display import display, HTML, clear_output, Javascript \n",
- "\n",
- "def fill_list(list_control,source):\n",
- " options_list = ['--Select--'] \n",
- " options_list.extend([s for s in source])\n",
- " list_control.options = options_list\n",
- "\n",
- "# URI panel\n",
- "uri_header = widgets.HTML(value=\"URI\", height='10%', width='100%')\n",
- "uri_select = widgets.Select(height='100%', width='99%')\n",
- "\n",
- "uri_select._css = (\n",
- " (None, 'height', '90%'),\n",
- " (None, 'width', '95%'),\n",
- " ('select', 'overflow-x', 'auto'),\n",
- " ('select', 'width', '100%'),\n",
- " ('select', 'margin', 0)\n",
- ")\n",
- "\n",
- "uri_box = widgets.Box(width='70%', height='100%')\n",
- "uri_box.children = [uri_header, uri_select]\n",
- "\n",
- "# Actions Panel\n",
- "actions_header = widgets.HTML(value=\" \", width='100%', height='10%')\n",
- "quick_text = widgets.Text(value='', width='100%', placeholder='Quick scoring')\n",
- "quick_text._css = (\n",
- " (None, 'width', '100%'),\n",
- ")\n",
- "rating_btn = widgets.RadioButtons(description='Rating:', options=['1', '2', '3'], width='100%')\n",
- "assign_btn = widgets.Button(description='Score', width='45%')\n",
- "assign_btn.button_style = 'primary'\n",
- "save_btn = widgets.Button(description='Save', width='45%')\n",
- "save_btn.button_style = 'primary'\n",
- "save_btn._css = (\n",
- " (None, 'margin-left', '10%'),\n",
- ")\n",
- "\n",
- "actions_box = widgets.Box(width='30%', height='100%')\n",
- "actions_box.children = [actions_header,quick_text,rating_btn, assign_btn,save_btn]\n",
- "\n",
- "scoring_form = widgets.HBox(width='90%', height=250)\n",
- "scoring_form.children = [uri_box, actions_box]\n",
- "\n",
- "\n",
- "def data_loader(): \n",
- " us_uris = []\n",
- "\n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f, delimiter='\\t')\n",
- " \n",
- " for row in reader: \n",
- " if row['fulluri'] not in us_uris and row['uri_sev'] == '0': \n",
- " us_uris.append(row['fulluri'])\n",
- "\n",
- " fill_list(uri_select,us_uris)\n",
- " uri_select.value = \"--Select--\" \n",
- "\n",
- "display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- "data_loader()\n",
- "display(scoring_form)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Update Suspicious Requests"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import csv\n",
- "import datetime\n",
- "import subprocess \n",
- "\n",
- "def assign_score(b):\n",
- " scored_threats = []\n",
- " clear_output()\n",
- " uri = quick_text.value or uri_select.value\n",
- " uri_sev = int(rating_btn.selected_label) if not \"--Select--\" in uri_select.value else \"\"\n",
- "\n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f, delimiter='\\t')\n",
- " rowct = 0\n",
- " with open(score_tmp, 'w') as score:\n",
- " wr = csv.DictWriter(score, delimiter='\\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames) \n",
- " wr.writeheader()\n",
- " \n",
- " for row in reader:\n",
- " if row['fulluri'] == uri:\n",
- " row['uri_sev'] = uri_sev\n",
- " scored_threats.append(row) \n",
- " rowct += 1\n",
- " try:\n",
- " wr.writerow(row)\n",
- " except:\n",
- " print str(row)\n",
- " \n",
- " #works on the feedback tab-separated file\n",
- " if not os.path.exists(score_fbk): \n",
- " with open(score_fbk, 'w') as feedback:\n",
- " wr = csv.DictWriter(feedback, delimiter='\\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames) \n",
- " wr.writeheader()\n",
- " \n",
- " with open(score_fbk, 'a') as feedback:\n",
- " for row in scored_threats:\n",
- " wr = csv.DictWriter(feedback, delimiter='\\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames) \n",
- " wr.writerow(row)\n",
- " \n",
- " clear_output()\n",
- " print \"{0} matching requests scored\".format(rowct)\n",
- " !mv $score_tmp $sconnect\n",
- " data_loader()\n",
- " uri_select.value = \"--Select--\"\n",
- " quick_text.value = \"\"\n",
- "\n",
- "\n",
- "def save(b): \n",
- " clear_output()\n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- " data_loader()\n",
- " display(scoring_form)\n",
- " display(Javascript('reloadParentData();')) \n",
- " ml_feedback()\n",
- " print \"Suspicious requests successfully updated\"\n",
- "\n",
- "\n",
- "assign_btn.on_click(assign_score)\n",
- "save_btn.on_click(save)\n",
- " \n",
- "\n",
- "def ml_feedback():\n",
- " dst_name = os.path.basename(sconnect)\n",
- " str_fb=\"DSOURCE={0} &&\\\n",
- " FDATE={1} &&\\\n",
- " source /etc/spot.conf &&\\\n",
- " usr=$(echo $LUSER | cut -f3 -d'/') &&\\\n",
- " mlnode=$MLNODE &&\\\n",
- " lpath=$LPATH &&\\\n",
- " scp {2} $usr@$mlnode:$lpath/{3}\".format(dsource,date,score_fbk,dst_name)\n",
- " subprocess.call(str_fb, shell=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "# !cp $sconnectbu $sconnect"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2.0
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.5"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/ipynb_templates/ThreatInvestigation.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/ThreatInvestigation.md b/spot-oa/oa/proxy/ipynb_templates/ThreatInvestigation.md
index 0681461..8c9fbfb 100644
--- a/spot-oa/oa/proxy/ipynb_templates/ThreatInvestigation.md
+++ b/spot-oa/oa/proxy/ipynb_templates/ThreatInvestigation.md
@@ -1,6 +1,6 @@
-#PROXY Threat Investigation Notebook
+# PROXY Threat Investigation Notebook
-###Dependencies
+### Dependencies
- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html)
- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/)
- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip)
@@ -20,83 +20,41 @@ The following python modules will have to be imported for the notebook to work c
from IPython.display import display, Javascript, clear_output
-##Pre-requisites
+## Pre-requisites
- Execution of the spot-oa process for Proxy
-- Score a set connections at the Edge Investigation Notebook
+- Correct installation of the UI [Read more](/ui/INSTALL.md)
+- Score a set connections at the Edge Investigation Notebook
- Correct setup of the spot.conf file. [Read more](/wiki/Edit%20Solution%20Configuration)
-##Additional Configuration
+## Additional Configuration
`top_results` - This value defines the number of rows that will be displayed onscreen after the expanded search.
This also affects the number of IPs that will appear in the Timeline chart.
-##Data
-The whole process in this notebook depends entirely on the existence of the scored _proxy_scores.tsv_ file, which is generated at the OA process, and scored at the Edge Investigation Notebook.
-
-**Input files**
-Schema for these files can be found [here](/spot-oa/oa/proxy)
-
- ~/spot-oa/data/proxy/<date>/proxy_scores.tsv
-
-**Output files**
-- threats.csv : Pipe separated file containing the comments saved by the user. This file is updated every time the user adds comments for a new threat.
-
- Schema with zero-indexed columns:
-
- 0.hash: string
- 1.title: string
- 2.description: string
-
-- incident-progression-\<anchor hash>.json : Json file generated in base of the results from the expanded
-search. This file includes a list of all requests performed to and from the URI under analysis, as well as the request methods used and the response content type.
-These results are limited to the day under analysis.
-this file will serve as datasource for the Incident Progression chart at the storyboard.
-
- Schema with zero-indexed columns:
-
- {
- 'fulluri':<URI under investigation>,
- 'requests': [{
- 'clientip':<client IP>,
- 'referer':<referer for the URI under analysis>,
- 'reqmethod':<method used to connect to the URI>,
- 'resconttype':<content type of the response>
- }, ...
- ],
- 'referer_for':[
- <List of unique URIs refered by the URI under investigation>
- ]
- }
-
-- timeline-\<anchor hash>.tsv : Tab separated file, this file lists all the client IP's that connected to the URI under investigation, including:
-the duration of the connection, response code and exact date and time of the connections.
-
- Schema with zero-indexed columns:
-
- 0.tstart: string
- 1.tend: string
- 2.duration: string
- 3.clientip: string
- 4.respcode: string
-
-- es-\<anchor hash>.tsv : (Expanded Search). Tab separated file, this is formed with the results from the Expanded Search query. Includes all connections where the investigated URI matches the `referer` or the `full_uri` columns.
+##Data source
+Data should exists in the following tables:
+ *proxy*
+ *proxy_threat_investigation*
-**HDFS tables consumed**
+**Output**
+The following tables will be populated after the threat investigation process:
+ *proxy_storyboard*
+ *proxy_timeline*
- proxy
+The following files will be created and stored in HDFS.
+ incident-progression-\<anchor hash>.json
-##Functions
+## Functions
**Widget configuration**
This is not a function, but more like global code to set up styles and widgets to format the output of the notebook.
`start_investigation():` - This function cleans the notebook from previous executions, then calls the data_loader() function to obtain the data and afterwards displays the corresponding widgets
-`data_loader():` - This function loads the source _proxy_scores.tsv_ file into a csv dictionary reader to create a list with all disctinct `full_uri` values
-where `uri_sev` = 1. This function will also read through the _threats.tsv_ file to discard all URIs that have already been investigated.
-
+`data_loader():` - This function lcalls the *threats* query to get the source and destination IP's previously scored as high risk to create a list with all distinct `full_uri` values.
+
`fill_list(list_control,source):` - This function populates a listbox widget with the given data list and appends an empty item at the top with the value '--Select--' (Just for visualization sake)
`display_controls():` - This function will only display the main widget box, containing:
@@ -106,9 +64,7 @@ where `uri_sev` = 1. This function will also read through the _threats.tsv_ file
- Container for the "Top N results" HTML table
`search_ip(b):` - This function is triggered by the _onclick_ event of the "Search" button.
-This will get the selected value from the listbox and perform a query to the _proxy_ table to retrieve all comunication involving the selected URI.
-Using MD5 algorythm, the URI will be hashed and use it in the name of the output files (anchor hash)
-The output of the query will automatically fill the es-/<anchor hash>.tsv file.
+This calls the graphql *threat / details* query to find additional connections involving the selected full uri.
Afterwards it will read through the output file to display the HTML table, this will be limited to the value set in the _top_results_ variable. At the same time, four dictionaries will be filled:
- clientips
- reqmethods *
@@ -119,8 +75,6 @@ Afterwards it will read through the output file to display the HTML table, this
This function will also display the 'Threat summary' and 'title' textboxes, along with the 'Save' button.
`save_threat_summary(b):` - This function is triggered by the _onclick_ event on the 'Save' button.
- This will take the contents of the form and create/update the _threats.csv_ file.
-
-`file_is_empty(path):` - Performs a validation to check the file size to determine if it is empty.
+Removes the widgets and cleans the notebook from previous executions, removes the selected value from the listbox widget and executes the *createStoryboard* mutation to save the data for the storyboard.
`removeWidget(index):` - Javascript function that removes a specific widget from the notebook.
\ No newline at end of file
[02/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/SuspiciousStore.js b/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
index 7c0bfe1..f79d697 100755
--- a/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
+++ b/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
@@ -1,182 +1,196 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-var d3 = require('d3');
-
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotUtils = require('../../../js/utils/SpotUtils');
-var ProxyConstants = require('../constants/ProxyConstants');
-var RestStore = require('../../../js/stores/RestStore');
-
-var ALL_FILTER = 'all';
-var URI_FILTER = 'fulluri';
-
-var CHANGE_FILTER_EVENT = 'change_filter';
-var HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
-var UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
-var SELECT_THREAT_EVENT = 'select_treath';
-
-var filterName = '';
-var highlightedThread = null;
-var selectedThread = null;
-
-var SuspiciousStore = assign(new RestStore(ProxyConstants.API_SUSPICIOUS), {
- _parser: d3.tsv,
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- headers: {
- p_date: 'Time',
- clientip: 'Client IP',
- host: 'Host',
- uri_rep: ' ',
- webcat: 'Web Category',
- respcode_name: 'Response Code'
- },
- ITERATOR: ['p_date', 'clientip', 'host', 'uri_rep', 'webcat', 'respcode_name'],
- getData: function () {
- var state, filter;
-
- filter = this.getFilter();
-
- if (!filter || !this._data) {
- state = this._data || {data: []};
- }
- else {
- state = assign(
- {},
- this._data
- );
-
- if (state.data) {
- state.data = state.data.filter((item) => {
- return filterName === URI_FILTER ?
- // Only look on URI field
- item[URI_FILTER].indexOf(filter) >= 0 :
- // Look on clientip and URI fields
- item.clientip == filter || item.fulluri.indexOf(filter) >= 0;
- });
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotUtils = require('../../../js/utils/SpotUtils');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const DATE_VAR = 'date';
+const URI_VAR = 'uri';
+const CLIENT_IP_VAR = 'clientIp';
+
+const CHANGE_FILTER_EVENT = 'change_filter';
+const HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
+const UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
+const SELECT_THREAT_EVENT = 'select_treath';
+
+class SuspiciousStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+
+ this.filterName = null;
+ this.highlightedThread = null;
+ this.selectedThread = null;
+
+ this.headers = {
+ datetime: 'Time',
+ clientip: 'Client IP',
+ host: 'Host',
+ uri_rep: ' ',
+ webcat: 'Web Category',
+ respcode_name: 'Response Code'
+ };
+
+ this.ITERATOR = ['datetime', 'clientip', 'host', 'uri_rep', 'webcat', 'respcode_name'];
+ }
+
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$uri:String,$clientIp:SpotIpType) {
+ proxy {
+ suspicious(date: $date, uri:$uri,clientIp:$clientIp) {
+ uriport: uriPort
+ uripath: uriPath
+ uriquery: uriQuery
+ datetime
+ network_context: networkContext
+ duration
+ useragent: userAgent
+ uri_rep: uriRep
+ score
+ username
+ webcat: webCategory
+ resconttype: responseContentType
+ host
+ referer
+ csbytes: clientToServerBytes
+ fulluri: uri
+ serverip: serverIp
+ reqmethod: requestMethod
+ respcode: responseCode
+ clientip: clientIp
+ respcode_name: responseCodeLabel
+ scbytes: serverToClientBytes
+ }
+ }
}
- }
+ `;
+ }
- if (state.data) {
- state.data = state.data.filter(function (item) {
- return item.uri_sev == "0";
- });
+ unboxData(data) {
+ return data.proxy.suspicious;
+ }
- if (state.data.length > SpotConstants.MAX_SUSPICIOUS_ROWS) state.data = state.data.slice(0, SpotConstants.MAX_SUSPICIOUS_ROWS);
- }
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
- return state;
- },
- setData: function (data) {
- this._data = data;
-
- this.emitChangeData();
- },
- setDate: function (date) {
- this.setEndpoint(ProxyConstants.API_SUSPICIOUS.replace('${date}', date.replace(/-/g, '')));
- },
- setFilter: function (filter) {
+ setFilter(filter) {
if (filter === '') {
- filterName = '';
- this.removeRestFilter(ALL_FILTER);
- this.removeRestFilter(URI_FILTER);
+ this.filterName = null;
+ this.unsetVariable(URI_VAR);
+ this.unsetVariable(CLIENT_IP_VAR);
}
else if (SpotUtils.IP_V4_REGEX.test(filter)) {
- this.removeRestFilter(URI_FILTER);
- this.setRestFilter(ALL_FILTER, filter);
- filterName = ALL_FILTER;
+ this.unsetVariable(URI_VAR, filter);
+ this.setVariable(CLIENT_IP_VAR, filter);
}
else {
- this.removeRestFilter(ALL_FILTER);
- this.setRestFilter(URI_FILTER, filter);
- filterName = URI_FILTER;
+ this.unsetVariable(CLIENT_IP_VAR);
+ this.setVariable(URI_VAR, filter);
}
- this.emitChangeFilter();
- },
- getFilter: function () {
- return this.getRestFilter(filterName);
- },
- emitChangeFilter: function () {
- this.emit(CHANGE_FILTER_EVENT);
- },
- addChangeFilterListener: function (callback) {
- this.on(CHANGE_FILTER_EVENT, callback);
- },
- removeChangeFilterListener: function (callback) {
+ this.notifyListeners(CHANGE_FILTER_EVENT);
+ }
+
+ getFilter() {
+ return this.getVariable(CLIENT_IP_VAR) || this.getVariable(URI_VAR) || '';
+ }
+
+ addChangeFilterListener(callback) {
+ this.addListener(CHANGE_FILTER_EVENT, callback);
+ }
+
+ removeChangeFilterListener(callback) {
this.removeListener(CHANGE_FILTER_EVENT, callback);
- },
- highlightThreat: function (threat) {
- highlightedThread = threat;
- this.emitHighlightThreat();
- },
- getHighlightedThreat: function () {
- return highlightedThread;
- },
- addThreatHighlightListener: function (callback) {
- this.on(HIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatHighlightListener: function (callback) {
+ }
+
+ highlightThreat(threat) {
+ this.highlightedThread = threat;
+ this.notifyListeners(HIGHLIGHT_THREAT_EVENT);
+ }
+
+ getHighlightedThreat() {
+ return this.highlightedThread;
+ }
+
+ addThreatHighlightListener(callback) {
+ this.addListener(HIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatHighlightListener(callback) {
this.removeListener(HIGHLIGHT_THREAT_EVENT, callback);
- },
- emitHighlightThreat: function () {
- this.emit(HIGHLIGHT_THREAT_EVENT);
- },
- unhighlightThreat: function () {
- highlightedThread = null;
- this.emitUnhighlightThreat();
- },
- addThreatUnhighlightListener: function (callback) {
- this.on(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatUnhighlightListener: function (callback) {
+ }
+
+ unhighlightThreat() {
+ this.highlightedThread = null;
+ this.notifyListeners(UNHIGHLIGHT_THREAT_EVENT);
+ }
+
+ addThreatUnhighlightListener(callback) {
+ this.addListener(UNHIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatUnhighlightListener(callback) {
this.removeListener(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- emitUnhighlightThreat: function () {
- this.emit(UNHIGHLIGHT_THREAT_EVENT);
- },
- selectThreat: function (threat) {
- selectedThread = threat;
- this.emitThreatSelect();
- },
- getSelectedThreat: function () {
- return selectedThread;
- },
- addThreatSelectListener: function (callback) {
- this.on(SELECT_THREAT_EVENT, callback);
- },
- removeThreatSelectListener: function (callback) {
+ }
+
+ selectThreat(threat) {
+ this.selectedThread = threat;
+ this.notifyListeners(SELECT_THREAT_EVENT);
+ }
+
+ getSelectedThreat() {
+ return this.selectedThread;
+ }
+
+ addThreatSelectListener(callback) {
+ this.addListener(SELECT_THREAT_EVENT, callback);
+ }
+
+ removeThreatSelectListener(callback) {
this.removeListener(SELECT_THREAT_EVENT, callback);
- },
- emitThreatSelect: function () {
- this.emit(SELECT_THREAT_EVENT);
}
-});
+}
+
+const ss = new SuspiciousStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_FILTER:
- SuspiciousStore.setFilter(action.filter);
+ ss.setFilter(action.filter);
break;
case SpotConstants.UPDATE_DATE:
- SuspiciousStore.setDate(action.date);
+ ss.setDate(action.date);
break;
case SpotConstants.RELOAD_SUSPICIOUS:
- SuspiciousStore.reload();
+ ss.sendQuery();
break;
case SpotConstants.HIGHLIGHT_THREAT:
- SuspiciousStore.highlightThreat(action.threat);
+ ss.highlightThreat(action.threat);
break;
case SpotConstants.UNHIGHLIGHT_THREAT:
- SuspiciousStore.unhighlightThreat();
+ ss.unhighlightThreat();
break;
case SpotConstants.SELECT_THREAT:
- SuspiciousStore.selectThreat(action.threat);
+ ss.selectThreat(action.threat);
break;
}
});
-module.exports = SuspiciousStore;
+module.exports = ss;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/TimelineStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/TimelineStore.js b/spot-oa/ui/proxy/js/stores/TimelineStore.js
index fa6c704..cf8d886 100755
--- a/spot-oa/ui/proxy/js/stores/TimelineStore.js
+++ b/spot-oa/ui/proxy/js/stores/TimelineStore.js
@@ -1,61 +1,83 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-const assign = require('object-assign');
-const d3 = require('d3');
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-const FlowConstants = require('../constants/ProxyConstants');
const SpotConstants = require('../../../js/constants/SpotConstants');
-const RestStore = require('../../../js/stores/RestStore');
-
-const FILTER_NAME = 'hash';
-
-const TimelineStore = assign(new RestStore(FlowConstants.API_TIMELINE), {
- _parser: d3.tsv,
- _date:'',
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(FlowConstants.API_TIMELINE.replace('${date}', date.replace(/-/g, '')));
- this._date = date;
- },
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+const URI_VAR = 'uri';
+
+class TimelineStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$uri:String!) {
+ proxy {
+ threat {
+ timeline(date:$date, uri:$uri) {
+ duration
+ clientip: clientIp
+ tend: endDatetime
+ respcode: responseCodeLabel
+ tstart: startDatetime
+ }
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.proxy.threat.timeline;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
getDate() {
- return this._date;
- },
- setFilter: function (value)
- {
- this.setRestFilter(FILTER_NAME, value);
- },
- getFilter: function ()
- {
- return this.getRestFilter(FILTER_NAME);
- },
- clearFilter: function ()
- {
- this.removeRestFilter(FILTER_NAME);
+ return this.getVariable(DATE_VAR);
}
-});
+ setUri(uri) {
+ this.setVariable(URI_VAR, uri);
+ }
+
+ getUri() {
+ return this.getVariable(URI_VAR);
+ }
+}
+
+const ts = new TimelineStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- TimelineStore.setDate(action.date);
-
+ ts.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- TimelineStore.clearFilter();
- TimelineStore.resetData();
+ ts.resetData();
break;
case SpotConstants.SELECT_COMMENT:
- TimelineStore.setFilter(action.comment.hash);
-
- TimelineStore.reload();
-
+ ts.setUri(action.comment.uri);
+ ts.sendQuery();
break;
}
});
-module.exports = TimelineStore;
+module.exports = ts;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/storyboard.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/storyboard.js b/spot-oa/ui/proxy/js/storyboard.js
index f642f30..8996fe1 100755
--- a/spot-oa/ui/proxy/js/storyboard.js
+++ b/spot-oa/ui/proxy/js/storyboard.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
require("babel-polyfill");
const React = require('react');
@@ -9,6 +24,12 @@ const SpotActions = require('../../js/actions/SpotActions');
const SpotConstants = require('../../js/constants/SpotConstants');
const SpotUtils = require('../../js/utils/SpotUtils');
const StoryboardActions = require('../../js/actions/StoryboardActions');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
ReactDOM.render(
(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/suspicious.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/suspicious.js b/spot-oa/ui/proxy/js/suspicious.js
index 95912c8..d38d11f 100755
--- a/spot-oa/ui/proxy/js/suspicious.js
+++ b/spot-oa/ui/proxy/js/suspicious.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -11,12 +26,22 @@ const SpotUtils = require('../../js/utils/SpotUtils');
// Build and Render Toolbar
const FilterInput = require('./components/FilterInput.react');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
function resetFilterAndReload() {
EdInActions.setFilter('');
EdInActions.reloadSuspicious();
};
+function switchComponents () {
+ SpotUtils.switchDivs(SpotConstants.DETAILS_PANEL, SpotConstants.SCORING_PANEL);
+};
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
ReactDOM.render(
(
<form className="form-inline">
@@ -56,11 +81,9 @@ const Panel = require('../../js/components/Panel.react');
const SuspiciousPanel = require('./components/SuspiciousPanel.react');
const NetworkViewPanel = require('./components/NetworkViewPanel.react');
-const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+const ScoreNotebook = require('./components/ScoreNotebook.react');
const DetailsPanel = require('./components/DetailsPanel.react');
-const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure(SpotConstants.NOTEBOOK_PANEL);
-
ReactDOM.render(
<div id="spot-content">
<PanelRow>
@@ -72,14 +95,18 @@ ReactDOM.render(
<NetworkViewPanel className="proxy-force" />
</Panel>
</PanelRow>
- <PanelRow>
- <Panel title={ipynbClosure.getTitle()} container extraButtons={ipynbClosure.getButtons}>
- <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="proxy/${date}/Edge_Investigation.ipynb" />
+ <div className="sortable">
+ <PanelRow title={SpotConstants.SCORING_PANEL}>
+ <Panel title={SpotConstants.SCORING_PANEL} reloadable switchable onReload={EdInActions.reloadSuspicious} onSwitch={switchComponents} className="col-md-12">
+ <ScoreNotebook />
</Panel>
- <Panel title={SpotConstants.DETAILS_PANEL} expandable>
- <DetailsPanel />
+ </PanelRow>
+ <PanelRow title={SpotConstants.DETAILS_PANEL}>
+ <Panel title={SpotConstants.DETAILS_PANEL} container switchable expandable onSwitch={switchComponents} className="col-md-12">
+ <DetailsPanel title={SpotConstants.DETAILS_PANEL} />
</Panel>
</PanelRow>
+ </div>
</div>,
document.getElementById('spot-content-wrapper')
);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/threat-investigation.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/threat-investigation.js b/spot-oa/ui/proxy/js/threat-investigation.js
index 208f2ee..0701a32 100755
--- a/spot-oa/ui/proxy/js/threat-investigation.js
+++ b/spot-oa/ui/proxy/js/threat-investigation.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -7,6 +22,12 @@ const SpotActions = require('../../js/actions/SpotActions');
const SpotUtils = require('../../js/utils/SpotUtils');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
ReactDOM.render(
(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/package.json b/spot-oa/ui/proxy/package.json
index bc3f4de..b986881 100755
--- a/spot-oa/ui/proxy/package.json
+++ b/spot-oa/ui/proxy/package.json
@@ -6,10 +6,12 @@
"watch-suspicious": "watchify js/suspicious.js -o js/suspicious.bundle.min.js -v -d",
"watch-threat-investigation": "watchify js/threat-investigation.js -o js/threat-investigation.bundle.min.js -v -d",
"watch-storyboard": "watchify js/storyboard.js -o js/storyboard.bundle.min.js -v -d",
- "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard",
+ "watch-notebooks": "watchify js/notebooks.js -o js/notebooks.bundle.min.js -v -d",
+ "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard && npm run build-notebooks",
"build-suspicious": "browserify js/suspicious.js | uglifyjs -cm > js/suspicious.bundle.min.js",
"build-threat-investigation": "browserify js/threat-investigation.js | uglifyjs -cm > js/threat-investigation.bundle.min.js",
- "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js"
+ "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js",
+ "build-notebooks": "browserify js/notebooks.js | uglifyjs -cm > js/notebooks.bundle.min.js"
},
"browserify": {
"transform": [
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/storyboard.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/storyboard.html b/spot-oa/ui/proxy/storyboard.html
index cabd387..01bdd94 100755
--- a/spot-oa/ui/proxy/storyboard.html
+++ b/spot-oa/ui/proxy/storyboard.html
@@ -21,6 +21,7 @@
<title>Proxy :: Storyboard</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -148,62 +149,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious Connects</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/suspicious.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/suspicious.html b/spot-oa/ui/proxy/suspicious.html
index 1d77322..29bea60 100755
--- a/spot-oa/ui/proxy/suspicious.html
+++ b/spot-oa/ui/proxy/suspicious.html
@@ -21,6 +21,7 @@
<title>Proxy :: Suspicious</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -30,6 +31,8 @@
<link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
<!-- Spot styles -->
<link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
<!-- Favicon -->
<link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png">
@@ -138,62 +141,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
@@ -217,6 +165,7 @@
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
<script type="application/javascript" src="../js/tooltip.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
<script type="application/javascript" src="js/suspicious.bundle.min.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/threat-investigation.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/threat-investigation.html b/spot-oa/ui/proxy/threat-investigation.html
index 98c3ca4..9493f98 100755
--- a/spot-oa/ui/proxy/threat-investigation.html
+++ b/spot-oa/ui/proxy/threat-investigation.html
@@ -21,6 +21,7 @@
<title>Proxy :: Threat Investigation</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -67,62 +68,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious Connects</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious DNS</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/README.md b/spot-setup/README.md
index 4a80c1b..318f95e 100644
--- a/spot-setup/README.md
+++ b/spot-setup/README.md
@@ -8,48 +8,73 @@ This document is intended for any developer or sysadmin in learning the technica
This information will help you to get started on contributing to the Apache Spot Setup repository. For information about installing and running Apache Spot go to our [Installation Guide](http://spot.apache.org/doc/).
-Spot-setup contains the scripts to setup HDFS for Apache Spot solution. It will create the folder and database structure needed to run Apache Spot on HDFS and HIVE respectively. Spot-setup is a component of Apache Spot and is executed in the initial configuration after Linux user creation and before Ingest installation.
+Spot-setup contains the scripts to setup HDFS for Apache Spot solution. It will create the folder and database structure needed to run Apache Spot on HDFS and Impala respectively. Spot-setup is a component of Apache Spot and is executed in the initial configuration after Linux user creation and before Ingest installation.
## Prerequisites
To collaborate and run spot-setup, it is required the following prerequisites:
- A running Hadoop cluster
- Linux user account created in all nodes with sudo privileges
+- Enabling HDFS Access Control Lists
## General Description
-The main script in the repository is **hdfs_setup.sh** which is responsible of loading environment variables, creating folders in Hadoop for the different use cases (flow, DNS or Proxy), create the Hive database, and finally execute hive query scripts that creates Hive tables needed to access netflow, dns and proxy data.
+The main script in the repository is **hdfs_setup.sh** which is responsible of loading environment variables, creating folders in Hadoop for the different use cases (flow, DNS or Proxy), create the Impala database, and finally execute Impala query scripts that creates Impala tables needed to access netflow, dns and proxy data.
## Environment Variables
**spot.conf** is the file storing the variables needed during the installation process including node assignment, User interface, Machine Learning and Ingest gateway nodes.
-This file also contains sources desired to be installed as part of Apache Spot, general paths for HDFS folders, Kerberos information and local paths in the Linux filesystem for the user as well as for machine learning, ipython, lda and ingest processes.
+This file also contains sources desired to be installed as part of Apache Spot, general paths for HDFS folders, and local paths in the Linux filesystem for the user as well as for machine learning, ipython, lda and ingest processes.
-To read more about these variables, please review the [wiki] (http://spot.incubator.apache.org/doc/#configuration).
+To read more about these variables, please review the [documentation](http://spot.incubator.apache.org/doc/#configuration).
## Database Query Scripts
spot-setup contains a script per use case, as of today, there is a table creation script for each DNS, flow and Proxy data.
-These HQL scripts are intended to be executed as a Hive statement and must comply HQL standards.
+These HQL scripts are intended to be executed as a Impala statement and must comply HQL standards.
-We want to create tables in Avro/Parquet format to get a faster query performance. This format is an industry standard and you can find more information about it on:
-- Avro is a data serialization system - https://avro.apache.org/
+We create tables using Parquet format to get a faster query performance. This format is an industry standard and you can find more information about it on:
- Parquet is a columnar storage format - https://parquet.apache.org/
-To get to Avro/parquet format we need a staging table to store CSV data temporarily for Flow and DNS. Then, run a Hive query statement to insert these text-formatted records into the Avro/parquet table. Hive will manage to convert the text data into the desired format. The staging table must be cleaned after loading data to Avro/parquet table for the next batch cycle. For Flow and DNS, a set of a staging (CSV) and a final (Avro/parquet) tables are needed for each data entity. For Proxy, only the Avro/parquet table is needed.
+To get to parquet format we need a staging table to store CSV data temporarily for Flow and DNS. Then, run an Impala query statement to insert these text-formatted records into the parquet table. Impala will manage to convert the text data into the desired format. The staging table must be cleaned after loading data to parquet table for the next batch cycle. For Flow and DNS, a set of a staging (CSV) and a final (parquet) tables are needed for each data entity. For Proxy, only the parquet table is needed.
#### Flow Tables
-- flow - Avro/parquet final table to store flow records
-- flow_tmp - Text table to store temporarily flow records in CSV format
+- flow
+- flow_tmp
+- flow_chords
+- flow_edge
+- flow_ingest_summary
+- flow_scores
+- flow_storyboard
+- flow_threat_investigation
+- flow_timeline
#### DNS Tables
-- dns - Avro/parquet final table to store DNS records
-- dns_tmp - Text table to store temporarily DNS records in CSV format
+- dns
+- dns_tmp
+- dns_dendro
+- dns_edge
+- dns_ingest_summary
+- dns_scores
+- dns_storyboard
+- dns_threat_dendro
+- dns_threat_investigation
#### Proxy Tables
-- proxy - Avro/parquet final table to store Proxy records
+- proxy
+- proxy_edge
+- proxy_ingest_summary
+- proxy_scores
+- proxy_storyboard
+- proxy_threat_investigation
+- proxy_timeline
+
+
+## Migrating OA Data to Spot 1.0
+
+Please review migration documentation [here](migration/README.md).
## Licensing
@@ -61,10 +86,15 @@ Create a pull request and contact the maintainers.
## Issues
+<<<<<<< HEAD
Report issues at the Apache Spot [issues] (https://issues.apache.org/jira/projects/SPOT/issues) page.
+=======
+- Create an [issue](https://issues.apache.org/jira/browse/SPOT-20?jql=project%20%3D%20SPOT).
+- Go to our Slack [channel](https://apachespot.slack.com/messages/general).
+>>>>>>> 2ebe57238614b6aa1162c6dc30758c516ae79273
## Maintainers
-- [Moises Valdovinos] (https://github.com/moy8011)
-- [Everardo Lopez Sandoval] (https://github.com/EverLoSa)
+- [Moises Valdovinos](https://github.com/moy8011)
+- [Everardo Lopez Sandoval](https://github.com/EverLoSa)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_dns_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_dns_avro_parquet.hql b/spot-setup/create_dns_avro_parquet.hql
deleted file mode 100755
index 105c386..0000000
--- a/spot-setup/create_dns_avro_parquet.hql
+++ /dev/null
@@ -1,37 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns (
- frame_time STRING,
- unix_tstamp BIGINT,
- frame_len INT,
- ip_dst STRING,
- ip_src STRING,
- dns_qry_name STRING,
- dns_qry_class STRING,
- dns_qry_type INT,
- dns_qry_rcode INT,
- dns_a STRING
-)
-PARTITIONED BY (y INT, m INT, d INT, h int)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/dns/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "DnsRecord"
- , "namespace" : "com.cloudera.accelerators.dns.avro"
- , "fields": [
- {"name": "frame_time", "type":["string", "null"]}
- , {"name": "unix_tstamp", "type":["bigint", "null"]}
- , {"name": "frame_len", "type":["int", "null"]}
- , {"name": "ip_dst", "type":["string", "null"]}
- , {"name": "ip_src", "type":["string", "null"]}
- , {"name": "dns_qry_name", "type":["string", "null"]}
- , {"name": "dns_qry_class", "type":["string", "null"]}
- , {"name": "dns_qry_type", "type":["int", "null"]}
- , {"name": "dns_qry_rcode", "type":["int", "null"]}
- , {"name": "dns_a", "type":["string", "null"]}
- ]
-}');
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_dns_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_dns_parquet.hql b/spot-setup/create_dns_parquet.hql
new file mode 100755
index 0000000..38025c6
--- /dev/null
+++ b/spot-setup/create_dns_parquet.hql
@@ -0,0 +1,163 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns (
+frame_time STRING,
+unix_tstamp BIGINT,
+frame_len INT,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+dns_a STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT,
+h TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro (
+unix_tstamp BIGINT,
+dns_a STRING,
+dns_qry_name STRING,
+ip_dst STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/dendro';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_edge (
+unix_tstamp BIGINT,
+frame_len BIGINT,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+dns_a STRING,
+hh INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_scores (
+frame_time STRING,
+unix_tstamp BIGINT,
+frame_len BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+ml_score FLOAT,
+tld STRING,
+query_rep STRING,
+hh INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard (
+ip_threat STRING,
+dns_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro (
+anchor STRING,
+total BIGINT,
+dns_qry_name STRING,
+ip_dst STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/threat_dendro';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_investigation (
+unix_tstamp BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+ip_sev INT,
+dns_sev INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/threat_investigation';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_flow_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_flow_avro_parquet.hql b/spot-setup/create_flow_avro_parquet.hql
deleted file mode 100755
index 34aa63b..0000000
--- a/spot-setup/create_flow_avro_parquet.hql
+++ /dev/null
@@ -1,72 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow (
- treceived STRING,
- unix_tstamp BIGINT,
- tryear INT,
- trmonth INT,
- trday INT,
- trhour INT,
- trminute INT,
- trsec INT,
- tdur FLOAT,
- sip STRING,
- dip STRING,
- sport INT,
- dport INT,
- proto STRING,
- flag STRING,
- fwd INT,
- stos INT,
- ipkt BIGINT,
- ibyt BIGINT,
- opkt BIGINT,
- obyt BIGINT,
- input INT,
- output INT,
- sas INT,
- das INT,
- dtos INT,
- dir INT,
- rip STRING
- )
-PARTITIONED BY (y INT, m INT, d INT, h int)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/flow/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "FlowRecord"
- , "namespace" : "com.cloudera.accelerators.flows.avro"
- , "fields": [
- {"name": "treceived", "type":["string", "null"]}
- , {"name": "unix_tstamp", "type":["long", "null"]}
- , {"name": "tryear", "type":["int", "null"]}
- , {"name": "trmonth", "type":["int", "null"]}
- , {"name": "trday", "type":["int", "null"]}
- , {"name": "trhour", "type":["int", "null"]}
- , {"name": "trminute", "type":["int", "null"]}
- , {"name": "trsec", "type":["int", "null"]}
- , {"name": "tdur", "type":["float", "null"]}
- , {"name": "sip", "type":["string", "null"]}
- , {"name": "sport", "type":["int", "null"]}
- , {"name": "dip", "type":["string", "null"]}
- , {"name": "dport", "type":["int", "null"]}
- , {"name": "proto", "type":["string", "null"]}
- , {"name": "flag", "type":["string", "null"]}
- , {"name": "fwd", "type":["int", "null"]}
- , {"name": "stos", "type":["int", "null"]}
- , {"name": "ipkt", "type":["bigint", "null"]}
- , {"name": "ibytt", "type":["bigint", "null"]}
- , {"name": "opkt", "type":["bigint", "null"]}
- , {"name": "obyt", "type":["bigint", "null"]}
- , {"name": "input", "type":["int", "null"]}
- , {"name": "output", "type":["int", "null"]}
- , {"name": "sas", "type":["int", "null"]}
- , {"name": "das", "type":["int", "null"]}
- , {"name": "dtos", "type":["int", "null"]}
- , {"name": "dir", "type":["int", "null"]}
- , {"name": "rip", "type":["string", "null"]}
- ]
-}');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_flow_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_flow_parquet.hql b/spot-setup/create_flow_parquet.hql
new file mode 100755
index 0000000..41c4819
--- /dev/null
+++ b/spot-setup/create_flow_parquet.hql
@@ -0,0 +1,195 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow (
+treceived STRING,
+unix_tstamp BIGINT,
+tryear INT,
+trmonth INT,
+trday INT,
+trhour INT,
+trminute INT,
+trsec INT,
+tdur FLOAT,
+sip STRING,
+dip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flag STRING,
+fwd INT,
+stos INT,
+ipkt BIGINT,
+ibyt BIGINT,
+opkt BIGINT,
+obyt BIGINT,
+input INT,
+output INT,
+sas INT,
+das INT,
+dtos INT,
+dir INT,
+rip STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT,
+h TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_chords (
+ip_threat STRING,
+srcip STRING,
+dstip STRING,
+ibyt BIGINT,
+ipkt BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/chords';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_edge (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flags STRING,
+tos INT,
+ibyt BIGINT,
+ipkt BIGINT,
+input BIGINT,
+output BIGINT,
+rip STRING,
+obyt BIGINT,
+opkt BIGINT,
+hh INT,
+mn INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_scores (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+ipkt INT,
+ibyt INT,
+opkt INT,
+obyt INT,
+ml_score FLOAT,
+rank INT,
+srcip_INTernal INT,
+dstip_INTernal INT,
+src_geoloc STRING,
+dst_geoloc STRING,
+src_domain STRING,
+dst_domain STRING,
+src_rep STRING,
+dst_rep STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard (
+ip_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_threat_investigation (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+srcport INT,
+dstport INT,
+score INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/threat_investigation';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline (
+ip_threat STRING,
+tstart STRING,
+tend STRING,
+srcip STRING,
+dstip STRING,
+proto STRING,
+sport INT,
+dport INT,
+ipkt BIGINT,
+ibyt BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/timeline';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_proxy_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_proxy_avro_parquet.hql b/spot-setup/create_proxy_avro_parquet.hql
deleted file mode 100755
index 3fd7ea8..0000000
--- a/spot-setup/create_proxy_avro_parquet.hql
+++ /dev/null
@@ -1,76 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy (
-p_date string,
-p_time string,
-clientip string,
-host string,
-reqmethod string,
-useragent string,
-resconttype string,
-duration int,
-username string,
-authgroup string,
-exceptionid string,
-filterresult string,
-webcat string,
-referer string,
-respcode string,
-action string,
-urischeme string,
-uriport string,
-uripath string,
-uriquery string,
-uriextension string,
-serverip string,
-scbytes int,
-csbytes int,
-virusid string,
-bcappname string,
-bcappoper string,
-fulluri string
-)
-PARTITIONED BY (
-y string,
-m string,
-d string,
-h string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/proxy/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "ProxyRecord"
- , "namespace" : "com.cloudera.accelerators.proxy.avro"
- , "fields": [
- {"name": "p_date", "type":["string", "null"]}
- , {"name": "p_time", "type":["string", "null"]}
- , {"name": "clientip", "type":["string", "null"]}
- , {"name": "host", "type":["string", "null"]}
- , {"name": "reqmethod", "type":["string", "null"]}
- , {"name": "useragent", "type":["string", "null"]}
- , {"name": "resconttype", "type":["string", "null"]}
- , {"name": "duration", "type":["int", "null"]}
- , {"name": "username", "type":["string", "null"]}
- , {"name": "authgroup", "type":["string", "null"]}
- , {"name": "exceptionid", "type":["string", "null"]}
- , {"name": "filterresult", "type":["string", "null"]}
- , {"name": "webcat", "type":["string", "null"]}
- , {"name": "referer", "type":["string", "null"]}
- , {"name": "respcode", "type":["string", "null"]}
- , {"name": "action", "type":["string", "null"]}
- , {"name": "urischeme", "type":["string", "null"]}
- , {"name": "uriport", "type":["string", "null"]}
- , {"name": "uripath", "type":["string", "null"]}
- , {"name": "uriquery", "type":["string", "null"]}
- , {"name": "uriextension", "type":["string", "null"]}
- , {"name": "serverip", "type":["string", "null"]}
- , {"name": "scbytes", "type":["int", "null"]}
- , {"name": "csbytes", "type":["int", "null"]}
- , {"name": "virusid", "type":["string", "null"]}
- , {"name": "bcappname", "type":["string", "null"]}
- , {"name": "bcappoper", "type":["string", "null"]}
- , {"name": "fulluri", "type":["string", "null"]}
- ]
-}');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_proxy_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_proxy_parquet.hql b/spot-setup/create_proxy_parquet.hql
new file mode 100755
index 0000000..f665dc2
--- /dev/null
+++ b/spot-setup/create_proxy_parquet.hql
@@ -0,0 +1,177 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy (
+p_date STRING,
+p_time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+authgroup STRING,
+exceptionid STRING,
+filterresult STRING,
+webcat STRING,
+referer STRING,
+respcode STRING,
+action STRING,
+urischeme STRING,
+uriport STRING,
+uripath STRING,
+uriquery STRING,
+uriextension STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+virusid STRING,
+bcappname STRING,
+bcappoper STRING,
+fulluri STRING
+)
+PARTITIONED BY (
+y STRING,
+m STRING,
+d STRING,
+h STRING
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+webcat STRING,
+respcode STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+referer STRING,
+uriport STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+hh INT,
+respcode_name STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+webcat STRING,
+referer STRING,
+respcode INT,
+uriport INT,
+uripath STRING,
+uriquery STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+word STRING,
+ml_score FLOAT,
+uri_rep STRING,
+respcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard (
+p_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_threat_investigation (
+tdate STRING,
+fulluri STRING,
+uri_sev INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/threat_investigation';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline (
+p_threat STRING,
+tstart STRING,
+tend STRING,
+duration BIGINT,
+clientip STRING,
+respcode STRING,
+respcodename STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/timeline';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/hdfs_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/hdfs_setup.sh b/spot-setup/hdfs_setup.sh
index 8eb9ae3..86a26c0 100755
--- a/spot-setup/hdfs_setup.sh
+++ b/spot-setup/hdfs_setup.sh
@@ -18,35 +18,52 @@
#
DSOURCES=('flow' 'dns' 'proxy')
-DFOLDERS=('binary' 'hive' 'stage')
+DFOLDERS=('binary'
+'stage'
+'hive'
+'hive/oa'
+'hive/oa/chords'
+'hive/oa/edge'
+'hive/oa/summary'
+'hive/oa/suspicious'
+'hive/oa/storyboard'
+'hive/oa/threat_investigation'
+'hive/oa/timeline'
+'hive/oa/dendro'
+'hive/oa/threat_dendro'
+)
+
+# Sourcing spot configuration variables
source /etc/spot.conf
-#
-# creating HDFS user's folder
-#
-hadoop fs -mkdir ${HUSER}
-hadoop fs -chown ${USER}:supergroup ${HUSER}
+# Creating HDFS user's folder
+sudo -u hdfs hdfs dfs -mkdir ${HUSER}
+sudo -u hdfs hdfs dfs -chown ${USER}:supergroup ${HUSER}
+sudo -u hdfs hdfs dfs -chmod 775 ${HUSER}
+# Creating HDFS paths for each use case
for d in "${DSOURCES[@]}"
do
echo "creating /$d"
- hadoop fs -mkdir ${HUSER}/$d
+ hdfs dfs -mkdir ${HUSER}/$d
for f in "${DFOLDERS[@]}"
do
echo "creating $d/$f"
- hadoop fs -mkdir ${HUSER}/$d/$f
+ hdfs dfs -mkdir ${HUSER}/$d/$f
done
+
+ # Modifying permission on HDFS folders to allow Impala to read/write
+ hdfs dfs -chmod -R 775 ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
done
-#
-# create hive tables
-#
-#configure / create catalog
-hive -e "CREATE DATABASE ${DBNAME}"
+# Creating Spot Database
+impala-shell -i ${IMPALA_DEM} -q "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
+# Creating Impala tables
for d in "${DSOURCES[@]}"
do
- hive -hiveconf huser=${HUSER} -hiveconf dbname=${DBNAME} -f create_${d}_avro_parquet.hql
+ impala-shell -i ${IMPALA_DEM} --var=huser=${HUSER} --var=dbname=${DBNAME} -c -f create_${d}_parquet.hql
done
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migrate_to_spot_1_0.py
----------------------------------------------------------------------
diff --git a/spot-setup/migrate_to_spot_1_0.py b/spot-setup/migrate_to_spot_1_0.py
new file mode 100755
index 0000000..1c00a4f
--- /dev/null
+++ b/spot-setup/migrate_to_spot_1_0.py
@@ -0,0 +1,102 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+from migration.utilities import util
+
+pipelines = sys.argv[1]
+old_oa_path = sys.argv[2]
+staging_db = sys.argv[3]
+hdfs_staging_path = sys.argv[4]
+dest_db = sys.argv[5]
+impala_daemon = sys.argv[6]
+
+
+# Execution example:
+#./migrate_old_data.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+
+ global old_oa_path
+ old_oa_path = old_oa_path.rstrip('/')
+ old_spot_path = os.path.split(old_oa_path)[0]
+ new_spot_path = os.path.split(cur_path)[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+
+ log.info('Old Spot path: {0}'.format(old_spot_path))
+ log.info('Old OA path: {0}'.format(old_oa_path))
+ log.info('New Spot path: {0}'.format(new_spot_path))
+ log.info('New OA path: {0}'.format(new_oa_path))
+
+
+ log.info("Executing hdfs_setup.sh for initial setup")
+ util.execute_cmd('./hdfs_setup.sh',log)
+
+ log.info("Updating /etc/spot.conf with new changes in Spot 1.0")
+ util.execute_cmd("migration/spot_conf_migration.py '/etc/spot.conf' '{0}/spot.conf'".format(cur_path),log)
+
+ log.info("Copy ingest_conf.json file from old folder code to new Spot location")
+ util.execute_cmd('cp {0}/spot-ingest/ingest_conf.json {1}/spot-ingest/ingest_conf.json'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Copy Spot OA configuration files to new Spot location")
+ util.execute_cmd('cp {0}/oa/components/iana/iana_config.json {1}/oa/components/iana/iana_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/reputation/reputation_config.json {1}/oa/components/reputation/reputation_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/nc/nc_config.json {1}/oa/components/nc/nc_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/data/engine.json {1}/oa/components/data/engine.json'.format(old_oa_path, new_oa_path),log)
+
+ log.info("Copy Spot OA context files to new Spot location")
+ util.execute_cmd('cp {0}/context/iploc.csv {1}/context/iploc.csv'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/context/ipranges.csv {1}/context/ipranges.csv'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/context/networkcontext_1.csv {1}/context/networkcontext_1.csv'.format(old_oa_path, new_oa_path),log)
+
+ log.info("Install browserify and uglifyjs using npm")
+ os.chdir('{0}/ui'.format(new_oa_path))
+ util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
+ util.execute_cmd('npm install',log)
+
+ log.info("Install python-devel using yum")
+ util.execute_cmd('sudo yum install python-devel -y',log)
+
+ log.info("Install requirements using pip")
+ os.chdir('{0}'.format(new_oa_path))
+ util.execute_cmd('sudo -H -E pip install -r requirements.txt',log)
+
+
+ log.info("Migrate data from pipelines")
+ l_pipelines = pipelines.split(',')
+ valid_pipelines = ('flow', 'dns', 'proxy')
+ if len(l_pipelines) > 0:
+ for pipe in l_pipelines:
+ if pipe in valid_pipelines:
+ log.info("Migrating {0} old data to new spot release".format(pipe))
+ os.chdir('{0}/spot-setup/migration'.format(new_spot_path))
+ util.execute_cmd("python migrate_old_{0}_data.py '{1}' '{2}' '{3}' '{4}' '{5}' ".format(pipe, old_oa_path, staging_db, hdfs_staging_path, dest_db, impala_daemon),log)
+ else:
+ log.error("Pipeline {0} is not valid. Valid pipelines are flow, dns or proxy".format(pipe))
+ else:
+ log.error("Pipeline arguments must be separated by commas")
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/migration/README.md b/spot-setup/migration/README.md
new file mode 100644
index 0000000..29314ef
--- /dev/null
+++ b/spot-setup/migration/README.md
@@ -0,0 +1,71 @@
+## OA Data Migration to Spot 1.0
+
+This document is intended for any developer or sysadmin who wants to migrate their existing OA data to Spot 1.0. In previous Spot releases, OA data was stored in CSV files in a given location in the server used for OA (specified in spot.conf during original installation). In Spot 1.0, OA data is stored in Impala tables. The purposes of these scripts are to migrate independently each use case (flow, proxy and dns) from those CSV files into the new Impala tables.
+
+This migration process is optional and only for those users who want to keep usable their OA data (scores, edges, chords, dendros, ingest summaries, storyboards, threat investigations and timelines) generated in the previous Spot version.
+
+### Requirements
+
+- You must run first new Spot setup installation to have the new tables created in Impala.
+- You must log in to the OA server and run these scripts from there.
+- You must run these scripts from the spot-setup/migration folder in the new Spot 1.0 location
+
+### CSV to Impala Tables Mapping
+
+**Flow**
+
+CSV File | Impala Table
+---------|-------
+flow_scores.csv | flow_scores
+chord-*.tsv | flow_chords
+edge-*.tsv | flow_edge
+is_*.csv | flow_ingest_summary
+threats.csv | flow_storyboard
+flow_scores.csv (only scored values) | flow_threat_investigation
+sbdet-*.tsv | flow_timeline
+
+**DNS**
+
+CSV File | Impala Table
+---------|-------
+flow_scores.csv | dns_scores
+edge-*.csv | dns_edge
+dendro-*.csv | dns_dendro
+threat-dendro-*.csv | dns_threat_dendro
+is_*.csv | dns_ingest_summary
+threats.csv | dns_storyboard
+dns_scores.csv (only scored values) | dns_threat_investigation
+
+**Proxy**
+
+CSV File | Impala Table
+---------|-------
+edge-*.csv | proxy_edge
+is_*.csv | proxy_ingest_summary
+proxy_scores.csv | proxy_scores
+threats.csv | proxy_storyboard
+proxy_scores.csv (only scored values) | proxy_threat_investigation
+timeline-*.csv | proxy_timeline
+
+### Data Flow
+
+There is a launch and single script that will migrate all specified pipelines. This process will read each of the CSV from the existing location and import data to Impala tables accordingly, creating first a staging database and tables to load the records in the CSV and then insert that data into the new Spot 1.0 tables. You must execute this migration process from the server where old Spot release is located. You may provide one pipeline or all (flow, dns and proxy) according to your needs and your existing data. At the end of each script, the old data pipeline folder will be moved from the original location to a backup folder. Staging tables and their respective HDFS paths will be removed.
+
+### Execution
+
+```python
+./migrate_to_spot_1_0.py PIPELINES OLD_OA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
+```
+
+where variables mean:
+- **PIPELINES** - Comma-separated list of the pipelines to be migrated
+- **OLD_OA_PATH** - Path to the old Spot-OA release directory in the local filesystem
+- **STAGING_DB_NAME** - Name of the staging database to be created to temporarily store these records
+- **STAGING_DB_HDFS_PATH** - HDFS path of the staging database to be created to temporarily store these records
+- **NEW_SPOT_IMPALA_DB** - Database name of the Spot 1.0 Impala tables. Use the same as in the spot.conf when the new Spot release was installed
+- **IMPALA_DAEMON** - Choose an Impala daemon to be used to run scripts' queries.
+
+Example:
+```python
+./migrate_to_spot_1_0.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+```
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/__init__.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/__init__.py b/spot-setup/migration/__init__.py
new file mode 100644
index 0000000..ecb1860
--- /dev/null
+++ b/spot-setup/migration/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/create_dns_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_dns_migration_tables.hql b/spot-setup/migration/create_dns_migration_tables.hql
new file mode 100644
index 0000000..9afe6ba
--- /dev/null
+++ b/spot-setup/migration/create_dns_migration_tables.hql
@@ -0,0 +1,113 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE DATABASE IF NOT EXISTS ${var:dbname};
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_dendro_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro_tmp (
+dns_a STRING,
+dns_qry_name STRING,
+ip_dst STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/dendro'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_edge_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_edge_tmp (
+frame_time STRING,
+frame_len STRING,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type STRING,
+dns_qry_rcode STRING,
+dns_a STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/edge'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_ingest_summary_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary_tmp (
+tdate STRING,
+total BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/summary'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_scores_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_scores_tmp (
+frame_time STRING,
+frame_len BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+ml_score FLOAT,
+tld STRING,
+query_rep STRING,
+hh INT,
+ip_sev INT,
+dns_sev INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING,
+unix_tstamp BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/scores'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_storyboard_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard_tmp (
+ip_threat STRING,
+dns_threat STRING,
+title STRING,
+text STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+LOCATION '${var:hpath}/dns/storyboard'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_threat_dendro_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro_tmp (
+total BIGINT,
+dns_qry_name STRING,
+ip_dst STRING,
+sev int
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/threat_dendro'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/create_flow_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_flow_migration_tables.hql b/spot-setup/migration/create_flow_migration_tables.hql
new file mode 100644
index 0000000..b200efb
--- /dev/null
+++ b/spot-setup/migration/create_flow_migration_tables.hql
@@ -0,0 +1,126 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE DATABASE IF NOT EXISTS ${var:dbname};
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_scores_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_scores_tmp (
+sev int,
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+ipkt INT,
+ibyt INT,
+opkt INT,
+obyt INT,
+score FLOAT,
+rank INT,
+srcIpInternal INT,
+destIpInternal INT,
+srcGeo STRING,
+dstGeo STRING,
+srcDomain STRING,
+dstDomain STRING,
+srcIP_rep STRING,
+dstIP_rep STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/flow/scores/'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_chords_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_chords_tmp (
+srcip STRING,
+dstip STRING,
+ibyt BIGINT,
+ipkt BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/flow/chords'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_edge_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_edge_tmp (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flags STRING,
+tos INT,
+ibyt BIGINT,
+ipkt BIGINT,
+input BIGINT,
+output BIGINT,
+rip STRING,
+obyt BIGINT,
+opkt BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/flow/edge'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_ingest_summary_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary_tmp (
+tdate STRING,
+total BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/flow/summary'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_storyboard_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard_tmp (
+ip_threat STRING,
+title STRING,
+text STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+LOCATION '${var:hpath}/flow/storyboard'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_timeline_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline_tmp (
+tstart STRING,
+tend STRING,
+srcip STRING,
+dstip STRING,
+proto STRING,
+sport INT,
+dport INT,
+ipkt BIGINT,
+ibyt BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/flow/timeline'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/create_proxy_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_proxy_migration_tables.hql b/spot-setup/migration/create_proxy_migration_tables.hql
new file mode 100644
index 0000000..fa92ab2
--- /dev/null
+++ b/spot-setup/migration/create_proxy_migration_tables.hql
@@ -0,0 +1,127 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE DATABASE IF NOT EXISTS ${var:dbname};
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_edge_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge_tmp (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+webcat STRING,
+respcode STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+referer STRING,
+uriport STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/proxy/edge'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_ingest_summary_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary_tmp (
+tdate STRING,
+total BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/proxy/summary'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_scores_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores_tmp (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+webcat STRING,
+referer STRING,
+respcode INT,
+uriport INT,
+uripath STRING,
+uriquery STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+word STRING,
+ml_score FLOAT,
+uri_rep STRING,
+uri_sev INT,
+respcode_name STRING,
+network_context STRING,
+score_hash STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/proxy/scores'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_storyboard_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard_tmp (
+p_threat STRING,
+title STRING,
+text STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+LOCATION '${var:hpath}/proxy/storyboard'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_timeline_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline_tmp (
+tstart STRING,
+tend STRING,
+duration BIGINT,
+clientip STRING,
+respcode STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/proxy/timeline'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_iana_rcode_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_iana_rcode_tmp (
+respcode STRING,
+respcode_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/proxy/iana_rcode'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
[07/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb b/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
index 5cd89db..8372bbe 100644
--- a/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
+++ b/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
@@ -26,17 +26,10 @@
"except ImportError:\n",
" from IPython.html import widgets\n",
"from IPython.display import display, HTML, clear_output, Javascript \n",
- "\n",
- "with open('/etc/spot.conf') as conf:\n",
- " for line in conf.readlines():\n",
- " if \"DBNAME=\" in line: DBNAME = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
- " elif \"IMPALA_DEM=\" in line: IMPALA_DEM = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
" \n",
"path = os.getcwd().split(\"/\") \n",
"date = path[len(path)-1] \n",
"dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
- "sconnect = dpath + 'proxy_scores.tsv' \n",
- "threat_f = dpath + \"threats.csv\"\n",
"anchor = ''\n",
"anchor_hash = ''\n",
"clientips = defaultdict(int)\n",
@@ -46,7 +39,8 @@
"refered = defaultdict(int)\n",
"requests = []\n",
"top_results = 20\n",
- "details_limit = 1000 "
+ "details_limit = 1000\n",
+ "proxy_comments = {}"
]
},
{
@@ -148,28 +142,48 @@
" clear_output() \n",
" c_uri = []\n",
" uri_sev=[]\n",
- "\n",
- " #discards threats already commented\n",
- " if os.path.isfile(threat_f) and not file_is_empty(threat_f):\n",
- " with open(threat_f, 'r') as th:\n",
- " t_read = csv.reader(th, delimiter='|')\n",
- " t_read.next()\n",
- " for row in t_read: \n",
- " if row[0] != '' : c_uri.append(row[0])\n",
- " \n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.reader(f, delimiter='\\t')\n",
- " reader.next()\n",
- " for row in reader:\n",
- " # \"p_date\":0 , \"p_time\":1, \"clientip\":2 , \"host\":3, \"reqmethod\":4 , \"useragent\":5 , \"resconttype\":6\n",
- " # , \"duration\":7, \"username\":8 , \"webcat\":9, \"referer\":10, \"respcode\":11, \"uriport\":12, \"uripath\":13\n",
- " # , \"uriquery\":14, \"serverip\":15, \"scbytes\":16 , \"csbytes\":17, \"fulluri\":18, \"word\":19\n",
- " #Forms a hash out of the anchor to use it as the file name\n",
- " if row[22] == '1': \n",
- " row_hash = md5.new(str(row[18])).hexdigest()\n",
- " if row[18] not in uri_sev and row_hash not in c_uri:\n",
- " uri_sev.append(row[18])\n",
- "\n",
+ " global proxy_comments\n",
+ " \n",
+ " response = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " proxy{\n",
+ " threats{\n",
+ " list(date:$date) {\n",
+ " score\n",
+ " uri\n",
+ " datetime\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " )\n",
+ " proxy_comments = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " proxy{\n",
+ " threats{\n",
+ " comments(date:$date) { \n",
+ " uri\n",
+ " title\n",
+ " text\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " )\n",
+ " proxy_comments = proxy_comments['data']['proxy']['threats']['comments']\n",
+ " if not 'errors' in response: \n",
+ " for row in response['data']['proxy']['threats']['list']: \n",
+ " if row['uri'] not in uri_sev and row['score'] == 1: \n",
+ " uri_sev.append(row['uri'])\n",
+ " else:\n",
+ " print \"An error ocurred: \" + response[\"errors\"][0][\"message\"]\n",
+ " \n",
" threat_title.value =\"<h4>Suspicious URI</h4>\"\n",
" \n",
" if len(uri_sev) == 0:\n",
@@ -191,95 +205,96 @@
" display(topBox) \n",
" \n",
" def search_ip(b): \n",
- " global anchor \n",
- " global anchor_hash\n",
- " global ir_f\n",
+ " global anchor\n",
+ " global expanded_results\n",
" anchor='' \n",
- " anchor_hash = ''\n",
" anchor = susp_select.value \n",
- " anchor_hash = md5.new(str(anchor)).hexdigest()\n",
" removeWidget(3)\n",
" removeWidget(2)\n",
" removeWidget(1) \n",
- " height=80 \n",
- " ir_f = dpath + 'es-' + anchor_hash + \".csv\" \n",
- " table = \"<table><th>TIME</th><th>CLIENT IP</th><th>USERNAME</th><th>DURATION</th> \\\n",
- " <th>FULL URI</th><th>WEB CATEGORY</th><th>RESPONSE CODE</th><th>REQUEST METHOD</th><th>USER AGENT</th> \\\n",
- " <th>MIME TYPE</th><th>REFERER</th><th>URI PORT</th><th>PROXY IP</th><th>SERVER BYTES</th><th>CLIENT BYTES</th>\"\n",
+ " height=80 \n",
" \n",
- " if not os.path.isfile(ir_f) or (os.path.isfile(ir_f) and file_is_empty(ir_f)):\n",
- " # time:0, clientip:1, username:2, duration:3, fullURI:4, webcat:5, respcode:6, reqmethod:7\n",
- " # useragent:8, resconttype: 9, referer: 10, uriport:11, serverip:12, scbytes:13, csbytes:14\n",
- " imp_query = (\"\\\"SELECT p_time, clientip, username, duration, fulluri, webcat, respcode, reqmethod,\\\n",
- " useragent, resconttype, referer, uriport, serverip, scbytes, csbytes FROM {0}.proxy\\\n",
- " WHERE y='{1}' AND m='{2}' AND d='{3}' AND (fulluri='{4}' OR referer ='{4}') ORDER BY p_time\\\"\") \n",
- " \n",
- " imp_query = imp_query.format(DBNAME,yy,mm,dd,anchor) \n",
- " !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
- " !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $ir_f\n",
- " \n",
- " clear_output() \n",
- " req_method = {}\n",
+ " expanded_results = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!,$uri:String!){\n",
+ " proxy{\n",
+ " threat{\n",
+ " details(date:$date,uri:$uri) {\n",
+ " username\n",
+ " webCategory\n",
+ " responseContentType\n",
+ " datetime\n",
+ " referer\n",
+ " clientToServerBytes\n",
+ " duration\n",
+ " userAgent\n",
+ " uri\n",
+ " serverIp\n",
+ " requestMethod\n",
+ " responseCode\n",
+ " uriPort\n",
+ " clientIp\n",
+ " serverToClientBytes\n",
+ " }\n",
+ " }\n",
+ " } \n",
+ " }\n",
+ " \"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'uri': anchor\n",
+ " }\n",
+ " )\n",
+ " \n",
+ " \n",
+ " if not 'errors' in expanded_results: \n",
+ " i = 0\n",
+ " table = \"<table><th>TIME</th><th>CLIENT IP</th><th>USERNAME</th><th>DURATION</th> \\\n",
+ " <th>FULL URI</th><th>WEB CATEGORY</th><th>RESPONSE CODE</th><th>REQUEST METHOD</th><th>USER AGENT</th> \\\n",
+ " <th>MIME TYPE</th><th>REFERER</th><th>URI PORT</th><th>PROXY IP</th><th>SERVER BYTES</th><th>CLIENT BYTES</th>\"\n",
+ " for row in expanded_results['data']['proxy']['threat']['details']:\n",
+ " if i < top_results:\n",
+ " table += \"<tr><td>\"+ str(row['datetime'])+\"</td><td>\"+str(row['clientIp'])+\"</td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['username'])+\"\\\n",
+ " </div></td><td>\"+str(row['duration'])+\"</td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['uri'])+\"</div>\\\n",
+ " </td><td>\"+str(row['webCategory'])+\"</td>\\\n",
+ " <td>\"+str(row['responseCode'])+\"</td><td>\"+str(row['requestMethod'])+\"</td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['userAgent'])+\"</div></td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['responseContentType'])+\"</div></td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['referer'])+\"</div></td>\\\n",
+ " <td>\"+str(row['uriPort'])+\"</td><td>\"+str(row['serverIp'])+\"</td><td>\\\n",
+ " \"+str(row['serverToClientBytes'])+\"</td><td>\"+str(row['clientToServerBytes'])+\"</td></tr>\"\n",
+ "\n",
+ " height += 20\n",
+ " i+=1\n",
+ " table += \"</table>\" \n",
+ " result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results)\n",
+ " else:\n",
+ " table = \"<table></table>\"\n",
+ " result_html_title.value='<h4>No results were found.</h4>'\n",
"\n",
- " with open(ir_f, 'r') as f:\n",
- " #Creates default dictionaries\n",
- " global reqmethods\n",
- " global rescontype\n",
- " global referers\n",
- " global refered\n",
- " global requests\n",
- " global clientips\n",
- " clientips = defaultdict(int)\n",
- " reqmethods = defaultdict(int)\n",
- " rescontype = defaultdict(int)\n",
- " referers = defaultdict(int)\n",
- " refered = defaultdict(int)\n",
- " try:\n",
- " reader = csv.reader(f, delimiter='\\t')\n",
- " reader.next() # Skip headers\n",
- " i=0 \n",
- " for row in reader:\n",
- " clientips[row[1]]+=1\n",
- " reqmethods[row[7]]+=1\n",
- " rescontype[row[9]]+=1\n",
- " if row[10] != anchor:\n",
- " #Source URI's that refered the user to the threat\n",
- " referers[row[10]]+=1\n",
- " if({'clientip':row[1],'referer':row[10],'reqmethod':row[7],'resconttype':row[9]}) not in requests:\n",
- " requests.append({'clientip':row[1],'referer':row[10],'reqmethod':row[7],'resconttype':row[9]})\n",
- " if i < top_results:\n",
- " table += \"<tr><td>\"+row[0]+\"</td><td>\"+row[1]+\"</td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[2]+\"</div></td><td>\"+row[3]+\"</td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[4]+\"</div></td><td>\"+row[5]+\"</td>\\\n",
- " <td>\"+row[6]+\"</td><td>\"+row[7]+\"</td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[8]+\"</div></td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[9]+\"</div></td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[10]+\"</div></td>\\\n",
- " <td>\"+row[11]+\"</td><td>\"+row[12]+\"</td><td>\"+row[13]+\"</td><td>\"+row[14]+\"</td></tr>\"\n",
- " else:\n",
- " #Destination URI's refered by the threat\n",
- " refered[row[4]]+=1\n",
- " height += 20\n",
- " i+=1\n",
- " table += \"</table>\" \n",
- " result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results)\n",
- " except:\n",
- " table = \"<table></table>\"\n",
- " result_html_title.value='<h4>No results were found.</h4>'\n",
- " \n",
- " result_html.value=table\n",
- " result_html_box.children = [result_html]\n",
+ " result_html.value=table\n",
+ " result_html_box.children = [result_html]\n",
" \n",
- " display_threat_box(anchor)\n",
- " resultTableBox.children = [result_html_title, result_html_box]\n",
- " display(bottomBox)\n",
+ " display_threat_box(anchor)\n",
+ " resultTableBox.children = [result_html_title, result_html_box]\n",
+ " display(bottomBox)\n",
+ " \n",
+ " \n",
" search_btn.on_click(search_ip)\n",
"\n",
" \n",
- "def display_threat_box(ip): \n",
+ "def display_threat_box(ip): \n",
+ " global expanded_results\n",
+ " global proxy_comments\n",
+ " \n",
+ " title =\"\" \n",
+ " comments = \"\"\n",
+ " title = next((item['title'] for item in proxy_comments if item.get(\"uri\") == ip), \"\")\n",
+ " comments = next((item['text'] for item in proxy_comments if item.get(\"uri\") == ip), \"\")\n",
" result_title.value=\"<h4 class='spot-text-wrapper spot-text-xlg' data-toggle='tooltip'>Threat summary for \" + anchor +\"</h4>\"\n",
- " tc_txt_title = widgets.Text(value='', placeholder='Threat Title', width='100%')\n",
- " tc_txa_summary = widgets.Textarea(value='', height=100, width='95%')\n",
+ " tc_txt_title = widgets.Text(value=title, placeholder='Threat Title', width='100%')\n",
+ " tc_txa_summary = widgets.Textarea(value=comments, height=100, width='95%')\n",
" tc_btn_save = widgets.Button(description='Save', width='65px', layout='width:100%')\n",
" tc_btn_save.button_style = 'primary'\n",
" \n",
@@ -293,75 +308,56 @@
" resultSummaryBox.children = [result_title,result_summary_box]\n",
" \n",
" def save_threat_summary(b):\n",
- " global anchor \n",
- " global anchor_hash \n",
- " if anchor != '': \n",
- " global threat_f\n",
- " if not os.path.exists(threat_f): \n",
- " with open(threat_f, 'w') as comment:\n",
- " comment.write('hash|title|summary\\n')\n",
- " \n",
- " with open(threat_f, 'a') as comment:\n",
- " comment.write(anchor_hash + '|' + tc_txt_title.value + '|' +\n",
- " tc_txa_summary.value.replace('\\n', '\\\\n') + '\\n') \n",
- " \n",
- " display(Javascript(\"$(\\\"option[data-value='\" + anchor +\"']\\\").remove();\")) \n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > .widget-box:gt(0)').remove();\"))\n",
- " \n",
- " response = \"Summary successfully saved\"\n",
- " incident_progression(anchor, anchor_hash)\n",
- " timeline(anchor, anchor_hash)\n",
+ " result_msg = \"\"\n",
+ " threat_title = tc_txt_title.value \n",
+ " threat_comment = tc_txa_summary.value\n",
+ "\n",
+ " \n",
+ " if anchor != '': \n",
+ " mutation=\"\"\"mutation(\n",
+ " $date: SpotDateType, \n",
+ " $uri: String!, \n",
+ " $text: String!, \n",
+ " $title: String!,\n",
+ " $threatDetails: [ProxyThreatDetailsInputType!]!,\n",
+ " $first:Int) \n",
+ " {\n",
+ " proxy {\n",
+ " createStoryboard(input:{\n",
+ " threatDetails: $threatDetails,\n",
+ " date: $date, \n",
+ " uri: $uri, \n",
+ " title: $title, \n",
+ " text: $text,\n",
+ " first:$first})\n",
+ " {success}\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'uri': anchor,\n",
+ " 'title': threat_title,\n",
+ " 'text': threat_comment,\n",
+ " 'threatDetails': expanded_results['data']['proxy']['threat']['details'],\n",
+ " 'first':top_results\n",
+ " }\n",
+ "\n",
+ " response = GraphQLClient.request(mutation,variables)\n",
+ " if not 'errors' in response:\n",
+ " start_investigation()\n",
+ " result_msg = \"Story board successfully created\"\n",
+ " else:\n",
+ " result_msg = response['errors'][0]['message'] \n",
" else:\n",
- " response = \"No data selected\" \n",
- " \n",
+ " result_msg = \"No data selected\" \n",
+ "\n",
" susp_select.selected_label = susp_select.options[0]\n",
- " display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
+ " display(widgets.Box((widgets.HTML(value=result_msg, width='100%'),)))\n",
" \n",
" tc_btn_save.on_click(save_threat_summary)\n",
- " \n",
- " \n",
- "def incident_progression(anchor, anchor_hash):\n",
- " file_name = dpath + 'incident-progression-'+anchor_hash+'.json'\n",
- " jsonstring = json.dumps({'fulluri':anchor, 'requests':requests,'referer_for':referers.keys()})\n",
- " if not os.path.exists(file_name):\n",
- " with open(file_name, 'w') as f:\n",
- " f.write(jsonstring) \n",
- " response = \"Incident progression successfuly created\"\n",
- " display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
- " \n",
- " \n",
- "def timeline(anchor, anchor_hash): \n",
- " response = \"\"\n",
- " susp_ips = []\n",
- " if clientips:\n",
- " srtlist = sorted(list(clientips.items()), key=lambda x: x[1], reverse=True)\n",
- " for val in srtlist[:top_results]:\n",
- " susp_ips.append(val[0]) \n",
- " \n",
- " if anchor != \"\":\n",
- " sbdet_f = dpath + \"timeline-\"+anchor_hash+\".tsv\"\n",
- " if not os.path.isfile(sbdet_f) or (os.path.isfile(sbdet_f) and file_is_empty(sbdet_f)): \n",
- " imp_query = \"\\\"SELECT concat(cast(p_date as string), ' ', cast(MIN(p_time) as string)) AS tstart,\\\n",
- " concat(cast(p_date as string), ' ', cast(MAX(p_time) as string)) AS tend, SUM(duration) AS duration,\\\n",
- " clientip, respcode from {0}.proxy WHERE fulluri='{1}' AND clientip IN ({5}) \\\n",
- " AND y='{2}' AND m='{3}' AND d='{4}' GROUP BY clientip, p_time, respcode, p_date ORDER BY clientip\\\n",
- " LIMIT {6}\\\"\"\n",
"\n",
- " imp_query=imp_query.format(DBNAME,anchor,yy,mm,dd,(\"'\" + \"','\".join(susp_ips) + \"'\"), details_limit) \n",
- " !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
- " !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $sbdet_f\n",
- " clear_output()\n",
- " \n",
- " response = \"Timeline successfully saved\"\n",
- " else:\n",
- " response = \"Timeline couldn't be created\"\n",
- " \n",
- " display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
- " data_loader()\n",
" \n",
- "def file_is_empty(path):\n",
- " return os.stat(path).st_size==0\n",
- "\n",
"def removeWidget(index):\n",
" js_command = \"$('.widget-area > .widget-subarea > .widget-box:eq({0})').remove();\".format(index) \n",
" display(Javascript(js_command)) "
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/proxy_conf.json
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_conf.json b/spot-oa/oa/proxy/proxy_conf.json
index 5378939..2228e80 100644
--- a/spot-oa/oa/proxy/proxy_conf.json
+++ b/spot-oa/oa/proxy/proxy_conf.json
@@ -42,14 +42,12 @@
, "serverip":15
, "scbytes":16
, "csbytes":17
- , "fulluri":18
+ , "fulluri":18
, "word":19
- , "score":20
+ , "ml_score":20
, "uri_rep":21
- , "uri_sev":22
- , "respcode_name":23
- , "network_context":24
- , "hash":25
+ , "respcode_name":22
+ , "network_context":23
},
"add_reputation":{
"fulluri":18
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/proxy_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_oa.py b/spot-oa/oa/proxy/proxy_oa.py
index 1324f1a..1fcef19 100644
--- a/spot-oa/oa/proxy/proxy_oa.py
+++ b/spot-oa/oa/proxy/proxy_oa.py
@@ -21,14 +21,18 @@ import json
import shutil
import sys
import datetime
-import csv, math
+import csv, math
from collections import OrderedDict
from utils import Util
from components.data.data import Data
from components.iana.iana_transform import IanaTransform
from components.nc.network_context import NetworkContext
+
+import api.resources.hdfs_client as HDFSClient
+import api.resources.impala_engine as impala
from multiprocessing import Process
import pandas as pd
+from impala.util import as_pandas
import time
import md5
@@ -68,7 +72,6 @@ class OA(object):
# initialize data engine
self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '')
- self._engine = Data(self._db, self._table_name,self._logger)
def start(self):
@@ -78,13 +81,12 @@ class OA(object):
####################
self._create_folder_structure()
+ self._clear_previous_executions()
self._add_ipynb()
self._get_proxy_results()
- self._add_reputation()
- self._add_severity()
+ self._add_reputation()
self._add_iana()
- self._add_network_context()
- self._add_hash()
+ self._add_network_context()
self._create_proxy_scores_csv()
self._get_oa_details()
self._ingest_summary()
@@ -102,12 +104,33 @@ class OA(object):
self._data_path,self._ingest_summary_path,self._ipynb_path = Util.create_oa_folders("proxy",self._date)
+ def _clear_previous_executions(self):
+
+ self._logger.info("Cleaning data from previous executions for the day")
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ table_schema = []
+ HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
+ table_schema=['suspicious', 'edge','threat_investigation', 'timeline', 'storyboard', 'summary' ]
+
+ for path in table_schema:
+ HDFSClient.delete_folder("{0}/{1}/hive/oa/{2}/y={3}/m={4}/d={5}".format(HUSER,self._table_name,path,yr,int(mn),int(dy)),user="impala")
+ impala.execute_query("invalidate metadata")
+
+ #removes Feedback file
+ HDFSClient.delete_folder("{0}/{1}/scored_results/{2}{3}{4}/feedback/ml_feedback.csv".format(HUSER,self._table_name,yr,mn,dy))
+ #removes json files from the storyboard
+ HDFSClient.delete_folder("{0}/{1}/oa/{2}/{3}/{4}/{5}".format(HUSER,self._table_name,"storyboard",yr,mn,dy))
+
+
+
def _add_ipynb(self):
if os.path.isdir(self._ipynb_path):
- self._logger.info("Adding edge investigation IPython Notebook")
- shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+ self._logger.info("Adding advanced mode IPython Notebook")
+ shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
self._logger.info("Adding threat investigation IPython Notebook")
shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
@@ -140,23 +163,23 @@ class OA(object):
self._logger.error("There was an error getting ML results from HDFS")
sys.exit(1)
- # add headers.
- self._logger.info("Adding headers")
- self._proxy_scores_headers = [ str(key) for (key,value) in self._conf['proxy_score_fields'].items() ]
-
self._proxy_scores = self._proxy_results[:]
def _create_proxy_scores_csv(self):
-
- proxy_scores_csv = "{0}/proxy_scores.tsv".format(self._data_path)
- proxy_scores_final = self._proxy_scores[:];
- proxy_scores_final.insert(0,self._proxy_scores_headers)
- Util.create_csv_file(proxy_scores_csv,proxy_scores_final, self._results_delimiter)
-
- # create bk file
- proxy_scores_bu_csv = "{0}/proxy_scores_bu.tsv".format(self._data_path)
- Util.create_csv_file(proxy_scores_bu_csv,proxy_scores_final, self._results_delimiter)
+ # get date parameters.
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ value_string = ""
+
+ for row in self._proxy_scores:
+ value_string += str(tuple(Util.cast_val(item) for item in row)) + ","
+
+ load_into_impala = ("""
+ INSERT INTO {0}.proxy_scores partition(y={2}, m={3}, d={4}) VALUES {1}
+ """).format(self._db, value_string[:-1], yr, mn, dy)
+ impala.execute_query(load_into_impala)
def _add_reputation(self):
@@ -195,25 +218,22 @@ class OA(object):
for result in rep_services_results:
rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
- self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
+ if rep_results:
+ self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
+ else:
+ self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
else:
self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
-
- def _add_severity(self):
- # Add severity column
- self._proxy_scores = [conn + [0] for conn in self._proxy_scores]
-
-
def _add_iana(self):
iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if os.path.isfile(iana_conf_file):
iana_config = json.loads(open(iana_conf_file).read())
proxy_iana = IanaTransform(iana_config["IANA"])
- proxy_rcode_index = self._conf["proxy_score_fields"]["respcode"]
- self._proxy_scores = [ conn + [ proxy_iana.get_name(conn[proxy_rcode_index],"proxy_http_rcode")] for conn in self._proxy_scores ]
+ proxy_rcode_index = self._conf["proxy_score_fields"]["respcode"]
+ self._proxy_scores = [ conn + [proxy_iana.get_name(conn[proxy_rcode_index],"proxy_http_rcode")] for conn in self._proxy_scores ]
else:
self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
@@ -225,96 +245,70 @@ class OA(object):
nc_conf = json.loads(open(nc_conf_file).read())["NC"]
proxy_nc = NetworkContext(nc_conf,self._logger)
ip_dst_index = self._conf["proxy_score_fields"]["clientip"]
- self._proxy_scores = [ conn + [proxy_nc.get_nc(conn[ip_dst_index])] for conn in self._proxy_scores ]
-
+ self._proxy_scores = [ conn + [proxy_nc.get_nc(conn[ip_dst_index])] for conn in self._proxy_scores ]
else:
self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
- def _add_hash(self):
- #A hash string is generated to be used as the file name for the edge files.
- #These fields are used for the hash creation, so this combination of values is treated as
- #a 'unique' connection
- cip_index = self._conf["proxy_score_fields"]["clientip"]
- uri_index = self._conf["proxy_score_fields"]["fulluri"]
- tme_index = self._conf["proxy_score_fields"]["p_time"]
-
- self._proxy_scores = [conn + [str( md5.new(str(conn[cip_index]) + str(conn[uri_index])).hexdigest() + str((conn[tme_index].split(":"))[0]) )] for conn in self._proxy_scores]
-
-
def _get_oa_details(self):
self._logger.info("Getting OA Proxy suspicious details")
# start suspicious connects details process.
p_sp = Process(target=self._get_suspicious_details)
p_sp.start()
-
- # p_sp.join()
+
def _get_suspicious_details(self):
- hash_list = []
+ uri_list = []
iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if os.path.isfile(iana_conf_file):
iana_config = json.loads(open(iana_conf_file).read())
proxy_iana = IanaTransform(iana_config["IANA"])
for conn in self._proxy_scores:
- conn_hash = conn[self._conf["proxy_score_fields"]["hash"]]
- if conn_hash not in hash_list:
- hash_list.append(conn_hash)
- clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
- fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
- date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
- if len(date) == 3:
- year=date[0]
- month=date[1].zfill(2)
- day=date[2].zfill(2)
- hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
- self._get_proxy_details(fulluri,clientip,conn_hash,year,month,day,hh,proxy_iana)
-
-
- def _get_proxy_details(self,fulluri,clientip,conn_hash,year,month,day,hh,proxy_iana):
-
- limit = 250
- output_delimiter = '\t'
- edge_file ="{0}/edge-{1}-{2}.tsv".format(self._data_path,clientip,conn_hash)
- edge_tmp ="{0}/edge-{1}-{2}.tmp".format(self._data_path,clientip,conn_hash)
-
- if not os.path.isfile(edge_file):
- proxy_qry = ("SELECT p_date, p_time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype, \
- referer, uriport, serverip, scbytes, csbytes, fulluri FROM {0}.{1} WHERE y=\'{2}\' AND m=\'{3}\' AND d=\'{4}\' AND \
- h=\'{5}\' AND fulluri =\'{6}\' AND clientip = \'{7}\' LIMIT {8};").format(self._db,self._table_name, year,month,day,hh,fulluri,clientip,limit)
-
- # execute query
- self._engine.query(proxy_qry,edge_tmp,output_delimiter)
- # add IANA to results.
+ clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
+ fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
+ date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
+ if len(date) == 3:
+ year=date[0]
+ month=date[1].zfill(2)
+ day=date[2].zfill(2)
+ hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
+ self._get_proxy_details(fulluri,clientip,year,month,day,hh,proxy_iana)
+
+
+ def _get_proxy_details(self,fulluri,clientip,year,month,day,hh,proxy_iana):
+ limit = 250
+ value_string = ""
+
+ query_to_load =("""
+ SELECT p_date, p_time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype,
+ referer, uriport, serverip, scbytes, csbytes, fulluri, {5} as hh
+ FROM {0}.{1} WHERE y='{2}' AND m='{3}' AND d='{4}' AND
+ h='{5}' AND fulluri='{6}' AND clientip='{7}' LIMIT {8};
+ """).format(self._db,self._table_name, year,month,day,hh,fulluri.replace("'","\\'"),clientip,limit)
+
+ detail_results = impala.execute_query(query_to_load)
+
+ if proxy_iana:
+ # add IANA to results.
self._logger.info("Adding IANA translation to details results")
- with open(edge_tmp) as proxy_details_csv:
- rows = csv.reader(proxy_details_csv, delimiter=output_delimiter,quotechar='"')
- next(proxy_details_csv)
- update_rows = [[conn[0]] + [conn[1]] + [conn[2]] + [conn[3]] + [conn[4]] + [proxy_iana.get_name(conn[5],"proxy_http_rcode") if proxy_iana else conn[5]] + [conn[6]] + [conn[7]] + [conn[8]] + [conn[9]] + [conn[10]] + [conn[11]] + [conn[12]] + [conn[13]] + [conn[14]] if len(conn) > 0 else [] for conn in rows]
- update_rows = filter(None, update_rows)
- header = ["p_date","p_time","clientip","host","webcat","respcode","reqmethod","useragent","resconttype","referer","uriport","serverip","scbytes","csbytes","fulluri"]
- update_rows.insert(0,header)
-
- # due an issue with the output of the query.
- update_rows = [ [ w.replace('"','') for w in l ] for l in update_rows ]
-
-
- # create edge file.
- self._logger.info("Creating edge file:{0}".format(edge_file))
- with open(edge_file,'wb') as proxy_details_edge:
- writer = csv.writer(proxy_details_edge, quoting=csv.QUOTE_NONE, delimiter=output_delimiter)
- if update_rows:
- writer.writerows(update_rows)
- else:
- shutil.copy(edge_tmp,edge_file)
-
- try:
- os.remove(edge_tmp)
- except OSError:
- pass
+
+ updated_rows = [conn + (proxy_iana.get_name(conn[5],"proxy_http_rcode"),) for conn in detail_results]
+ updated_rows = filter(None, updated_rows)
+ else:
+ updated_rows = [conn + ("") for conn in detail_results ]
+
+ for row in updated_rows:
+ value_string += str(tuple(item for item in row)) + ","
+
+ if value_string != "":
+ query_to_insert=("""
+ INSERT INTO {0}.proxy_edge PARTITION (y={1}, m={2}, d={3}) VALUES ({4});
+ """).format(self._db,year, month, day, value_string[:-1])
+ impala.execute_query(query_to_insert)
+
def _ingest_summary(self):
# get date parameters.
@@ -328,44 +322,36 @@ class OA(object):
result_rows = []
df_filtered = pd.DataFrame()
- ingest_summary_file = "{0}/is_{1}{2}.csv".format(self._ingest_summary_path,yr,mn)
- ingest_summary_tmp = "{0}.tmp".format(ingest_summary_file)
-
- if os.path.isfile(ingest_summary_file):
- df = pd.read_csv(ingest_summary_file, delimiter=',')
- #discards previous rows from the same date
- df_filtered = df[df['date'].str.contains("{0}-{1}-{2}".format(yr, mn, dy)) == False]
- else:
- df = pd.DataFrame()
-
# get ingest summary.
- ingest_summary_qry = ("SELECT p_date, p_time, COUNT(*) as total "
- " FROM {0}.{1}"
- " WHERE y='{2}' AND m='{3}' AND d='{4}' "
- " AND p_date IS NOT NULL AND p_time IS NOT NULL "
- " AND clientip IS NOT NULL AND p_time != '' "
- " AND host IS NOT NULL AND fulluri IS NOT NULL "
- " GROUP BY p_date, p_time;")
-
- ingest_summary_qry = ingest_summary_qry.format(self._db,self._table_name, yr, mn, dy)
- results_file = "{0}/results_{1}.csv".format(self._ingest_summary_path,self._date)
- self._engine.query(ingest_summary_qry,output_file=results_file,delimiter=",")
+
+ query_to_load=("""
+ SELECT p_date, p_time, COUNT(*) as total
+ FROM {0}.{1} WHERE y='{2}' AND m='{3}' AND d='{4}'
+ AND p_date IS NOT NULL AND p_time IS NOT NULL
+ AND clientip IS NOT NULL AND p_time != ''
+ AND host IS NOT NULL AND fulluri IS NOT NULL
+ GROUP BY p_date, p_time;
+ """).format(self._db,self._table_name, yr, mn, dy)
- if os.path.isfile(results_file):
- df_results = pd.read_csv(results_file, delimiter=',')
-
+ results = impala.execute_query(query_to_load)
+
+ if results:
+ df_results = as_pandas(results)
#Forms a new dataframe splitting the minutes from the time column/
df_new = pd.DataFrame([["{0} {1}:{2}".format(val['p_date'], val['p_time'].split(":")[0].zfill(2), val['p_time'].split(":")[1].zfill(2)), int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df_results.iterrows()],columns = ingest_summary_cols)
-
+ value_string = ''
#Groups the data by minute
sf = df_new.groupby(by=['date'])['total'].sum()
df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})
- df_final = df_filtered.append(df_per_min, ignore_index=True)
- df_final.to_csv(ingest_summary_tmp,sep=',', index=False)
-
- os.remove(results_file)
- os.rename(ingest_summary_tmp,ingest_summary_file)
+ df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False)
+ if len(df_final) > 0:
+ query_to_insert=("""
+ INSERT INTO {0}.proxy_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
+ """).format(self._db, yr, mn, dy, tuple(df_final))
+
+ impala.execute_query(query_to_insert)
+
else:
self._logger.info("No data found for the ingest summary")
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/start_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/start_oa.py b/spot-oa/oa/start_oa.py
index b604aab..660814f 100644
--- a/spot-oa/oa/start_oa.py
+++ b/spot-oa/oa/start_oa.py
@@ -20,6 +20,7 @@
import argparse
import os
import sys
+sys.path.append("../")
import logging
from utils import Util
@@ -39,43 +40,43 @@ def main():
start_oa(args)
def start_oa(args):
-
- # setup the main logger for all the OA process.
+
+ # setup the main logger for all the OA process.
logger = Util.get_logger('OA',create_file=False)
- logger.info("-------------------- STARTING OA ---------------------")
- validate_parameters_values(args,logger)
+ logger.info("-------------------- STARTING OA ---------------------")
+ validate_parameters_values(args,logger)
# create data type instance.
module = __import__("{0}.{0}_oa".format(args.type),fromlist=['OA'])
-
- # start OA.
+
+ # start OA.
oa_process = module.OA(args.date,args.limit,logger)
oa_process.start()
-
+
def validate_parameters_values(args,logger):
-
+
logger.info("Validating input parameter values")
#date.
- is_date_ok = True if len(args.date) == 8 else False
+ is_date_ok = True if len(args.date) == 8 else False
# type
dirs = os.walk(script_path).next()[1]
is_type_ok = True if args.type in dirs else False
-
- #limit
+
+ #limit
try:
int(args.limit)
is_limit_ok = True
except ValueError:
is_limit_ok = False
-
- if not is_date_ok: logger.error("date parameter is not correct, please validate it")
+
+ if not is_date_ok: logger.error("date parameter is not correct, please validate it")
if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1)
-
+
if __name__=='__main__':
- main()
\ No newline at end of file
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/utils.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 99b006e..2bed10e 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -121,6 +121,18 @@ class Util(object):
writer.writerows(content)
+ @classmethod
+ def cast_val(self,value):
+ try:
+ val = int(value)
+ except:
+ try:
+ val = float(value)
+ except:
+ val = str(value)
+ return val
+
+
class SecHead(object):
def __init__(self, fp):
self.fp = fp
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/requirements.txt
----------------------------------------------------------------------
diff --git a/spot-oa/requirements.txt b/spot-oa/requirements.txt
index 312360b..9f3afb8 100644
--- a/spot-oa/requirements.txt
+++ b/spot-oa/requirements.txt
@@ -5,7 +5,22 @@ Tld == 0.7.6
# spot-ui dependencies
jsonschema
-ipython == 3.2.0
pyzmq
jinja2
tornado
+mistune
+pygments
+terminado
+ipython == 3.2.1
+
+# GraphQL API dependencies
+flask
+flask-graphql
+graphql-core
+urllib3
+
+# API Resources
+setuptools>=3.4.4
+thrift==0.9.3
+impyla
+hdfs
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/runIpython.sh
----------------------------------------------------------------------
diff --git a/spot-oa/runIpython.sh b/spot-oa/runIpython.sh
index 04a09d9..38a4121 100755
--- a/spot-oa/runIpython.sh
+++ b/spot-oa/runIpython.sh
@@ -1,5 +1,4 @@
-#!/bin/sh
-
+#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,5 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-ipython notebook --no-mathjax --profile=ia --port=8889 --ip=0.0.0.0 --no-browser '--NotebookApp.extra_static_paths=["ui/ipython/"]'> ipython.out 2>&1&
+pushd $(dirname $0) > /dev/null
+SPOT_OA_DIR=$(pwd)
+popd > /dev/null
+IPYTHONDIR="$SPOT_OA_DIR/ipython/"
+export IPYTHONDIR
+ipython notebook --profile=spot > ipython.out 2>&1&
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/ui/README.md b/spot-oa/ui/README.md
index 4b35c86..d068b39 100755
--- a/spot-oa/ui/README.md
+++ b/spot-oa/ui/README.md
@@ -68,12 +68,15 @@ Provides tools for interactive visualization, noise filters, white listing, and
+ [Suspicious](#suspicious-4)
+ [Storyboard](#storyboard-4)
+ [Ingest Summary](#ingest-summary-2)
+ + [Notebook](#notebook-1)
* [DNS Stores](#dns-stores)
+ [Suspicious](#suspicious-5)
+ [Storyboard](#storyboard-5)
+ + [Notebook](#notebook-2)
* [Proxy Stores](#proxy-stores)
+ [Suspicious](#suspicious-6)
+ [Storyboard](#storyboard-6)
+ + [Notebook](#notebook-3)
## Intended Audience
@@ -91,7 +94,7 @@ Here you will find useful information to get you started on how to contribute to
## Technical Documentation
-Our UI is build on top of Flux application architecture principles, having an understanding of this pattern is highly recommended, for more information about Flux, please go to [Flux web page](https://facebook.github.io/flux/docs/overview.html).
+Our UI is built on top of Flux application architecture principles, having an understanding of this pattern is highly recommended, for more information about Flux, please go to [Flux web page](https://facebook.github.io/flux/docs/overview.html).
### ReactJS + Flux
@@ -104,12 +107,13 @@ For more information about ReactJS and Flux, please go to:
From now on we assume you are familiar with ReactJS+Flux applications.
-For every path found on this document, "SPOT" refers to the path where Spot UI is intalled and "PIPELINE" the pipeline subfolder you want to work on.
+For every path found on this document, "SPOT" refers to the path where Spot UI is installed and "PIPELINE" the pipeline subfolder you want to work on.
#### Development/Debugging process
1. Install Spot UI. Follow this [guide](INSTALL.md#install-spot-ui).
2. Run Spot UI. Follow this [guide](INSTALL.md#how-to-run-spot-ui).
+ 1. Set SPOT_DEV env var to 1 to enable GraphiQL UI and run ipython in debug mode.
3. Start watching for code changes
1. $ cd SPOT/ui/PIPELINE/
2. Watch one of the following modules
@@ -1034,6 +1038,13 @@ _SPOT/ui/js/flow/stores/_
Extends [RestStore.js](#reststore.js)
+###### Notebook
+
+spot-oa/ui/flow/js/stores/NotebookStore.js
+
+spot-oa/ui/flow/js/components/ScoreNotebook.react.js
+
+
##### DNS Stores
_SPOT/ui/js/dns/stores/_
@@ -1095,6 +1106,12 @@ _SPOT/ui/js/dns/stores/_
> - SELECT_COMMENT
> - RELOAD_COMMENTS
+###### Notebook
+
+spot-oa/ui/dns/js/stores/NotebookStore.js
+
+spot-oa/ui/dns/js/components/ScoreNotebook.react.js
+
##### Proxy Stores
_SPOT/ui/js/proxy/_
@@ -1153,3 +1170,9 @@ _SPOT/ui/js/proxy/_
> - UPDATE_DATE
> - SELECT_COMMENT
> - RELOAD_COMMENTS
+
+###### Notebook
+
+spot-oa/ui/proxy/js/stores/NotebookStore.js
+
+spot-oa/ui/proxy/js/components/ScoreNotebook.react.js
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/bootstrap-spot.min.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/bootstrap-spot.min.css b/spot-oa/ui/css/bootstrap-spot.min.css
index 6d05096..cbd2297 100644
--- a/spot-oa/ui/css/bootstrap-spot.min.css
+++ b/spot-oa/ui/css/bootstrap-spot.min.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
/*! Generated by Live LESS Theme Customizer */
.label,sub,sup{vertical-align:baseline}
body,figure{margin:0}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/dendrogram.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/dendrogram.css b/spot-oa/ui/css/dendrogram.css
index 06782df..0395c08 100644
--- a/spot-oa/ui/css/dendrogram.css
+++ b/spot-oa/ui/css/dendrogram.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
.dendrogram {
overflow-y: auto;
overflow-x: auto;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/main.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/main.css b/spot-oa/ui/css/main.css
index 00368ca..8421f13 100755
--- a/spot-oa/ui/css/main.css
+++ b/spot-oa/ui/css/main.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
/* All devices */
.spot-maximized {
width: 100% !important;
@@ -47,6 +63,7 @@
.spot-frame > .spot-frame-content > .panel > .panel-body > .panel-body-container {
width: 100%;
overflow-x: auto;
+ height: 1px;
}
.spot-frame .panel-body-container.container-box > * {
@@ -57,6 +74,39 @@
border: none;
}
+.sortable {
+ display: flex;
+ flex-flow: column;
+ height: 100%;
+}
+
+.overflowing {
+ overflow-y: scroll;
+}
+
+#accordion {
+ height: 85%;
+}
+
+.fit {
+ height: 50%;
+}
+
+::-webkit-scrollbar {
+ width: 7px;
+ height: 7px;
+}
+
+::-webkit-scrollbar-track {
+ -webkit-box-shadow: inset 0 0 6px rgba(0,0,0,0.3);
+ border-radius: 10px;
+}
+
+::-webkit-scrollbar-thumb {
+ border-radius: 10px;
+ -webkit-box-shadow: inset 0 0 10px rgba(0,0,0,0.5);
+}
+
/* Extra small devices (phones, less than 768px) */
/* Small devices (tablets, 768px and up) */
@@ -291,3 +341,24 @@ prevent the active panel from collapsing
.spot-text-wrapper.spot-text-lg {
width: 18em;
}
+
+/* GLYPHICON INPUT */
+
+.inner-addon {
+ position: relative;
+}
+
+.inner-addon .glyphicon {
+ position: absolute;
+ padding: 10px;
+ pointer-events: none;
+}
+
+.left-addon .glyphicon { left: 0px;}
+.right-addon .glyphicon { right: 0px;}
+
+.right-addon input { padding-right: 30px; }
+
+.margin-up-down {
+ padding: 20px 0px;
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/ipython_notebook.html b/spot-oa/ui/dns/ipython_notebook.html
new file mode 100644
index 0000000..21e4572
--- /dev/null
+++ b/spot-oa/ui/dns/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>DNS :: Advanced mode</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Font awesome ICONS-->
+ <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+
+ <style>
+ .spot-row {
+ height: 100%;
+ }
+
+ td.srcIP_rep, td.dstIP_rep {
+ white-space: nowrap;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: DNS :: Advanced mode</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+ <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js b/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
index 462e040..ad40124 100755
--- a/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
@@ -1,5 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ContentLoaderMixin = require('../../../js/components/ContentLoaderMixin.react');
@@ -29,7 +43,7 @@ const DetailsDendrogramPanel = React.createClass({
state.data = {
id: 'root',
- name: DendrogramStore.getSrcIp(),
+ name: DendrogramStore.getClientIp(),
children: []
};
@@ -45,7 +59,7 @@ const DetailsDendrogramPanel = React.createClass({
};
state.data.children.push(childNode);
- answers = item.dns_a.split('|');
+ answers = item.dns_a;
if (answers.length) {
let childId = 0;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsPanel.react.js b/spot-oa/ui/dns/js/components/DetailsPanel.react.js
index 550c516..15fc621 100755
--- a/spot-oa/ui/dns/js/components/DetailsPanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js b/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
index 59c5e23..0ed697e 100755
--- a/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
@@ -30,7 +45,6 @@ var DetailsTablePanel = React.createClass({
},
_render_dns_a_cell: function (answers)
{
- answers = (answers || "").split('|');
answers = answers.map(function (answer, idx)
{
return (
@@ -42,9 +56,6 @@ var DetailsTablePanel = React.createClass({
return answers;
},
- _render_dns_qry_class_cell: false,
- _render_dns_qry_rcode_cell: false,
- _render_dns_qry_type_cell: false,
_onChange: function ()
{
this.replaceState(DetailsStore.getData());
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/FilterInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/FilterInput.react.js b/spot-oa/ui/dns/js/components/FilterInput.react.js
index 666fc3e..3599603 100755
--- a/spot-oa/ui/dns/js/components/FilterInput.react.js
+++ b/spot-oa/ui/dns/js/components/FilterInput.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js b/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
index da17da7..ca322cc 100755
--- a/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
+++ b/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
@@ -8,8 +23,8 @@ const DendrogramMixin = require('../../../js/components/DendrogramMixin.react');
const IncidentProgressionStore = require('../stores/IncidentProgressionStore');
const fieldMapper = {
- ip_dst: 'dns_qry_name',
- dns_qry_name: 'ip_dst'
+ clientIp: 'dnsQuery',
+ dnsQuery: 'clientIp'
};
const IncidentProgressionPanel = React.createClass({
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js b/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
index 7328c76..d2ab8bd 100755
--- a/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
+++ b/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
@@ -46,7 +61,7 @@ function getNodesFromData(data) {
nodes[id] = {
id: id,
label: item[field],
- internalIp: field==='tld',
+ internalIp: field==='tld' ? 1 : 0,
hits: 1
};
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/ScoreNotebook.react.js b/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
new file mode 100644
index 0000000..e6c5259
--- /dev/null
+++ b/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
@@ -0,0 +1,206 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var SuspiciousStore = require('../stores/SuspiciousStore');
+var NotebookStore = require('../stores/NotebookStore');
+var EdInActions = require('../../../js/actions/EdInActions');
+var SelectInput = require('../../../js/components/scoring/SelectInput.react');
+var FilterSelectInput = require('../../../js/components/scoring/FilterSelectInput.react');
+var ButtonsScoring = require('../../../js/components/scoring/ButtonsScoring.react');
+var RatingInput = require('../../../js/components/scoring/RatingInput.react');
+var SearchGlobalInput = require('../../../js/components/scoring/SearchGlobalInput.react');
+var ScoreMessage = require('../../../js/components/scoring/ScoreMessage.react');
+
+var SpotUtils = require('../../../js/utils/SpotUtils');
+
+var ScoreNotebook = React.createClass({
+ // mixins: [GridPanelMixin],
+ emptySetMessage: 'There is no data available for selected date.',
+ propTypes: {
+ date: React.PropTypes.string.isRequired,
+ },
+ getInitialState: function () {
+ return {
+ scoredEmelents: []
+ };
+ },
+ componentDidMount: function() {
+ SuspiciousStore.addChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ this.setState({size: NotebookStore.completeClass})
+ },
+ componentWillUnmount: function () {
+ SuspiciousStore.removeChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ },
+ render: function () {
+ var content, state, data, cssCls, dnsNameArr = [], dstIpArr = [];
+ state = this.state || {};
+
+ if (state.error)
+ {
+ content = (
+ <div className="text-center text-danger">
+ {state.error}
+ </div>
+ );
+ }
+ else if (state.loading)
+ {
+ content = (
+ <div className="spot-loader">
+ Loading <span className="spinner"></span>
+ </div>
+ );
+ }
+ else if (!state.data || state.data.length === 0)
+ {
+ content = (
+ <div className="text-center">
+ {this.emptySetMessage || ''}
+ </div>
+ );
+ }
+ else
+ {
+ state.data.map((obj) => {
+ if(dstIpArr.indexOf(obj.ip_dst) === -1) {
+ dstIpArr.push(obj.ip_dst);
+ }
+ if(dnsNameArr.indexOf(obj.dns_qry_name) === -1) {
+ dnsNameArr.push(obj.dns_qry_name);
+ }
+ });
+
+ data = [
+ {value: 1, name: 'High', radioName: 'optradio', selected: true},
+ {value: 2, name: 'Medium', radioName: 'optradio', selected: false},
+ {value: 3, name: 'Low', radioName: 'optradio', selected: false}
+ ];
+
+ content = (
+ <div>
+ <div className="margin-up-down">
+ <SearchGlobalInput col="6" maxlength="255"/>
+ <RatingInput data={data} col="6"/>
+ </div>
+ <div className="margin-up-down">
+ <ButtonsScoring name="Score" action="score" onChange={this.score.bind(this)} col="3"/>
+ <ButtonsScoring name="Save" action="save" onChange={this.save.bind(this)} col="3"/>
+ <ButtonsScoring name="Reset Scoring" action="reset" onChange={this.reset.bind(this)} col="3"/>
+ </div>
+ <div className="margin-up-down">
+ <FilterSelectInput nameBox="Client Ip" idSelect="#dstIp" idInput="dstIpIn" col="3"/>
+ <FilterSelectInput nameBox="Query" idSelect="#query" idInput="queryIn" col="9"/>
+ </div>
+ <div className="margin-up-down">
+ <SelectInput title="Client IP" who="dstIp" options={dstIpArr} col="3"/>
+ <SelectInput title="Query" who="query" options={dnsNameArr} col="9"/>
+ </div>
+ <div className="margin-up-down">
+ <ScoreMessage who="scoreMsg"/>
+ </div>
+ </div>
+ );
+ }
+ cssCls = this.state.size ? 'col-md-6 col-lg-6 col-xs-12' : 'col-md-offset-3 col-lg-offset-3 col-md-6 col-lg-6 col-xs-12';
+
+ return(
+ <div className={cssCls + ' spot-frame'}>
+ {content}
+ </div>
+ )
+ },
+ _onChange: function() {
+ const data = SuspiciousStore.getData();
+ this.setState(data);
+ },
+ reset: function() {
+ swal({
+ title: 'Are you sure?',
+ text: "You won't be able to revert this!",
+ type: 'warning',
+ showCancelButton: true,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Yes, reset all!'
+ }).then(() => {
+ EdInActions.resetScoring(SpotUtils.getCurrentDate());
+ swal({
+ title: 'Done!',
+ text: "All scores have been reset.",
+ type: 'success',
+ showCancelButton: false,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Ok!'
+ }).then(() => {
+ this.setState({loading: true});
+ });
+ });
+ },
+ save: function() {
+ let variables = [];
+
+ if (this.state.scoredEmelents.length === 0) {
+ swal('Warning.','You need to score at least 1 threat.','warning');
+ } else {
+ this.state.scoredEmelents.map((row) => {
+ variables.push({
+ 'date': SpotUtils.getCurrentDate(),
+ 'clientIp': row[0] || null,
+ 'dnsQuery': row[1] || null,
+ 'score': row[2]
+ });
+ });
+
+ EdInActions.saveScoring(variables);
+ $('#scoreMsg').addClass("hidden");
+ this.setState({scoredEmelents: [], loading: true});
+ }
+ },
+ score: function() {
+ //this should be changed to take all data at the time, time needed.
+ let dataScored = this.state.scoredEmelents || [];
+ let quickIpScoring = document.getElementById('globalTxt').value;
+ let query = document.getElementById('query').value;
+ let dstIp = document.getElementById('dstIp').value;
+ let rating = $('input[name="optradio"]:checked').val();
+
+ //callback from the father
+ if(quickIpScoring !== '') {
+ dataScored.push([dstIp, quickIpScoring, rating]);
+ } else {
+ dataScored.push([dstIp, query, rating]);
+ }
+
+ this.removeSelected([dstIp, quickIpScoring,query]);
+ $('#scoreMsg').removeClass("hidden");
+ this.setState({scoredEmelents: dataScored});
+ },
+ removeSelected: function(data) {
+ //when an user score, all elements selected need to be removed.
+ data.map((element) => data.map((e) => e !== '' ? $(`option[value="${e}"]`).remove() : ''));
+ $(".select-picker, #globalTxt").val('');
+ }
+
+});
+
+
+module.exports = ScoreNotebook;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js b/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
index 951319e..a2d106f 100755
--- a/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
+++ b/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
@@ -26,28 +41,6 @@ var SuspiciousPanel = React.createClass({
</p>
);
},
- /**
- * Answers cell can have multiple pipe separated values. To allow
- * proper displaying of information lets place each answer inside
- * a block element let browsers decide how to display them.
- */
- _render_dns_a_cell: function (answers, item, idx)
- {
- var cellBody;
-
- answers = (answers || []).split('|');
-
- cellBody = answers.map(function (answer, i)
- {
- return (
- <div key={'answer_' + idx + '_' + i}>
- {answer}
- </div>
- );
- }.bind(this));
-
- return cellBody;
- },
_render_ip_dst_cell: function (ip_dst, item, idx)
{
var ip_dst_info, iconClass;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/constants/DnsConstants.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/constants/DnsConstants.js b/spot-oa/ui/dns/js/constants/DnsConstants.js
deleted file mode 100755
index 9e269fa..0000000
--- a/spot-oa/ui/dns/js/constants/DnsConstants.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var DnsConstants = {
- // API URLS
- API_SUSPICIOUS: '../../data/dns/${date}/dns_scores.csv',
- API_DETAILS: '../../data/dns/${date}/edge-${dns_qry_name}_${time}.csv',
- API_VISUAL_DETAILS: '../../data/dns/${date}/dendro-${ip_dst}.csv',
- API_COMMENTS: '../../data/dns/${date}/threats.csv',
- API_INCIDENT_PROGRESSION: '../../data/dns/${date}/threat-dendro-${id}.csv'
-};
-
-module.exports = DnsConstants;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/notebooks.js b/spot-oa/ui/dns/js/notebooks.js
new file mode 100644
index 0000000..81d923e
--- /dev/null
+++ b/spot-oa/ui/dns/js/notebooks.js
@@ -0,0 +1,72 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+ (
+ <form className="form-inline">
+ <div className="form-group">
+ <label htmlFor="dataDatePicker">Data Date:</label>
+ <div className="input-group input-group-xs">
+ <DateInput id="dataDatePicker" />
+ <div className="input-group-addon">
+ <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+ </div>
+ </div>
+ </div>
+ </form>
+ ),
+ document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+ <div id="spot-content">
+ <PanelRow maximized>
+ <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+ <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="dns/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+ </Panel>
+ </PanelRow>
+ </div>,
+ document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/CommentsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/CommentsStore.js b/spot-oa/ui/dns/js/stores/CommentsStore.js
index 291af68..47e2eea 100755
--- a/spot-oa/ui/dns/js/stores/CommentsStore.js
+++ b/spot-oa/ui/dns/js/stores/CommentsStore.js
@@ -1,33 +1,73 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-var d3 = require('d3');
-
-var DnsConstants = require('../constants/DnsConstants');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var RestStore = require('../../../js/stores/RestStore');
-
-var CommentsStore = assign(new RestStore(DnsConstants.API_COMMENTS), {
- _parser: d3.dsv('|', 'text/plain'),
- errorMessages: {
- 404: 'Please choose a different date, no comments have been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(DnsConstants.API_COMMENTS.replace('${date}', date.replace(/-/g, '')));
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+
+class CommentsStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType) {
+ dns {
+ threats{
+ comments(date:$date) {
+ title
+ summary: text
+ ...QueryComment
+ ...ClientIpComment
+ }
+ }
+ }
+ }
+
+ fragment QueryComment on DnsQueryCommentType {
+ dnsQuery
+ }
+
+ fragment ClientIpComment on DnsClientIpCommentType {
+ clientIp
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.dns.threats.comments;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+}
+
+const cs = new CommentsStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- CommentsStore.setDate(action.date);
+ cs.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- CommentsStore.reload();
+ cs.sendQuery();
break;
}
});
-module.exports = CommentsStore;
+module.exports = cs;
[16/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/common/__init__.py
----------------------------------------------------------------------
diff --git a/spot-ingest/common/__init__.py b/spot-ingest/common/__init__.py
index e69de29..ecb1860 100755
--- a/spot-ingest/common/__init__.py
+++ b/spot-ingest/common/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/pipelines/__init__.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/__init__.py b/spot-ingest/pipelines/__init__.py
index 8b13789..ecb1860 100644
--- a/spot-ingest/pipelines/__init__.py
+++ b/spot-ingest/pipelines/__init__.py
@@ -1 +1,16 @@
-
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/pipelines/dns/__init__.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/dns/__init__.py b/spot-ingest/pipelines/dns/__init__.py
index e69de29..ecb1860 100755
--- a/spot-ingest/pipelines/dns/__init__.py
+++ b/spot-ingest/pipelines/dns/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/pipelines/dns/load_dns_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/dns/load_dns_avro_parquet.hql b/spot-ingest/pipelines/dns/load_dns_avro_parquet.hql
index 730c544..93a363a 100755
--- a/spot-ingest/pipelines/dns/load_dns_avro_parquet.hql
+++ b/spot-ingest/pipelines/dns/load_dns_avro_parquet.hql
@@ -1,3 +1,18 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
SET hiveconf:data_location;
SET hiveconf:y;
SET hiveconf:m;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/pipelines/flow/__init__.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/flow/__init__.py b/spot-ingest/pipelines/flow/__init__.py
index e69de29..ecb1860 100755
--- a/spot-ingest/pipelines/flow/__init__.py
+++ b/spot-ingest/pipelines/flow/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/pipelines/flow/load_flow_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/flow/load_flow_avro_parquet.hql b/spot-ingest/pipelines/flow/load_flow_avro_parquet.hql
index 4fa9db0..c12f982 100755
--- a/spot-ingest/pipelines/flow/load_flow_avro_parquet.hql
+++ b/spot-ingest/pipelines/flow/load_flow_avro_parquet.hql
@@ -1,3 +1,18 @@
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
SET hiveconf:data_location;
SET hiveconf:y;
SET hiveconf:m;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/pipelines/proxy/__init__.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/proxy/__init__.py b/spot-ingest/pipelines/proxy/__init__.py
index e69de29..ecb1860 100644
--- a/spot-ingest/pipelines/proxy/__init__.py
+++ b/spot-ingest/pipelines/proxy/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ingest/pipelines/proxy/bluecoat.py
----------------------------------------------------------------------
diff --git a/spot-ingest/pipelines/proxy/bluecoat.py b/spot-ingest/pipelines/proxy/bluecoat.py
index de9ca6c..31d89ca 100644
--- a/spot-ingest/pipelines/proxy/bluecoat.py
+++ b/spot-ingest/pipelines/proxy/bluecoat.py
@@ -121,7 +121,7 @@ def save_data(rdd,sqc,db,db_table,topic):
sqc.setConf("hive.exec.dynamic.partition", "true")
sqc.setConf("hive.exec.dynamic.partition.mode", "nonstrict")
hive_table = "{0}.{1}".format(db,db_table)
- df.write.saveAsTable(hive_table,format="parquet",mode="append",partitionBy=('y','m','d','h'))
+ df.write.format("parquet").mode("append").insertInto(hive_table)
else:
print("------------------------LISTENING KAFKA TOPIC:{0}------------------------".format(topic))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/DATA_SAMPLE.md
----------------------------------------------------------------------
diff --git a/spot-ml/DATA_SAMPLE.md b/spot-ml/DATA_SAMPLE.md
new file mode 100644
index 0000000..2ffd064
--- /dev/null
+++ b/spot-ml/DATA_SAMPLE.md
@@ -0,0 +1,57 @@
+
+# DNS Labeled Data Set
+
+An IXIA BreakingPoint box was used to simulate both normal and attack (DNS tunnelling) DNS traffic. The resulting pcaps were obtained and fields relevant to Apache Spot (incubating) were ingested and stored in parquet format. The attacks can be differentiated from the normal activity due to codes that were inserted into the Transaction ID field (upon ingestion: ‘dns_id’) which identifies either the fact that the traffic was normal or identifies the specific DNS tunneling activity being used. We provide the data schema as well as the location and specifications of the data within Amazon-S3. Information is also provided for how to interpret the dns_id field.
+
+
+
+## Data Schema
+
+The schema for this data includes one field (called 'dns_id') in addition to what is usually used for DNS data in Apache Spot (incubating). The schema is as follows (see: http://www.iana.org/assignments/dns-parameters/dns-parameters.xhtml#dns-parameters-6 for more information):
+
+
+| Name | Type | Description |
+|--------------|:---------:|:-----------:|
+| frame_time | string | Time of packet capture (UTC) |
+| unix_tstamp | bigint | Time of packet capture (UNIX time) |
+| frame_len | int | Entire packet length |
+| ip_dst | string | IP address making the DNS query |
+| ip_src | string | IP address of the DNS server |
+| dns_qry_name | string | Resource record being queried, ex: 'google.com' |
+| dns_qry_class| string | Class of DNS record, ex: '0x00000001' (for Internet) |
+| dns_qry_type | int | Type of resource record, ex: 1 (for a host address) |
+| dns_qry_rcode| int | Error code for the results of the query, ex: 0 (for No Error)
+| dns_a | string | Answer to the query |
+| dns_id | string | Hexidecimal code inserting as the transaction ID used to differentiate normal queries from tunnelling (more details below) |
+
+## Interpreting dns_id
+The value of dns_id indicates that either the data row was taken from a packet capture of simulated normal DNS traffic, or from a packet capture of a particular type of simulated DNS tunnelling.
+
+Within BreakingPoint, Transaction IDs are represented as a decimal number. However, tshark dissects the transaction ID in its hexadecimal representation (the format contained within parenthesis in the table below).
+
+Within Apache Spot (incubating), only responses from DNS servers are ingested since the response packet contains the query made by the client and the response from the server in the same packet.
+
+
+| Super Flow Name | Transaction ID | Description |
+|---------------------------|:------------------:|-------------|
+| Brandon_DNS_domain_Test | 1008 (0x000003f0) | [Normal] This super flow simulates normal DNS queries.|
+| DNS_Tunnel_BE_1 | 1002 (0x000003ea) | [Attack] This super flow simulates a message being tunneled over DNS via the query name field (url's are random strings), with a ip address response (drawn from a file of randomly generated IPs) being sent via the DNS answer field. |
+| DNS_Tunnel_BE_2 | 1003 (0x000003eb}) | [Attack] This super flow simulates a message being tunneled over DNS via the query name field (url's random strings), with a response being given as no such url found. |
+| TCP_DNS_Tunnel_BE_1 | 1001 (0x000003e9) | [Attack] This super Flow simulates tunneling random noise using TCP over DNS. The payload is generated by a Markov Dictionary and encoded in the DNS requests (responses) by using hex0x20Hack encoding. |
+| TCP_DNS_Tunnel_BE_2 | 1005 (0x000003ed) | [Attack] This super Flow simulates tunneling random noise using TCP over DNS. The payload is generated by a Markov Dictionary and encoded in the DNS requests (responses) by using Base16Alpha encoding. |
+| TCP_DNS_Tunnel_BE_3 | 1007 (0x000003ef) | [Attack] This super Flow simulates tunneling random noise using TCP over DNS. The payload is generated by a Markov Dictionary and encoded in the DNS requests (responses) by using Base63 encoding. |
+
+## Data Location
+
+| File | Description | Location | Size |
+|-------------------|:------|:---------:|------:|
+| 20170509_parquet.tar.gz | Tarball of directory (parquet files) simulated May 9, 2017 | https://s3-us-west-2.amazonaws.com/apachespot/public_data_sets/dns_labeled_data/20170509_parquet.tar.gz | 5.3G |
+| 20170509_DATA_SPEC.md | Mark down document (the one you are reading now) | https://s3-us-west-2.amazonaws.com/apachespot/public_data_sets/dns_labeled_data/20170509_DATA_SPEC.md | 4K |
+
+## Number of Rows associated to each Value of dns_id in the Data File
+
+| Simulation Date | Total Records | dns_id=1008 | dns_id=1002 | dns_id=1003 | dns_id=1001 | dns_id=1005 | dns_id=1007 |
+|:-----------------|:--------------:|:-----------:|:-----------:|:-----------:|:-----------:|:-----------:|:-----------:|
+| 5/9/2017 | 391,364,387 | 391,314,477 | 16,317 | 21,666 | 4,156 | 2,743 | 5,028 |
+
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/INSTALL.md
----------------------------------------------------------------------
diff --git a/spot-ml/INSTALL.md b/spot-ml/INSTALL.md
index 59e4b06..103c3d6 100644
--- a/spot-ml/INSTALL.md
+++ b/spot-ml/INSTALL.md
@@ -14,7 +14,7 @@ Names and language that we will use from the configuration variables for Spot (t
- MLNODE The node from which the spot-ml routines are invoked
- HUSER An HDFS user path that will be the base path for the solution; this is usually the same user that you created to run the solution
- HPATH Location for storing intermediate results of the analysis on HDFS.
-- USER_DOMAIN Web domain associated to the user's network (for the DNS suspicous connects analysis). For example: USER_DOMAIN='intel'.
+- USER_DOMAIN Web domain associated to the user's network (for the DNS suspicious connects analysis). For example: USER_DOMAIN='intel'.
### Prepare data for input
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/ML_OPS.md
----------------------------------------------------------------------
diff --git a/spot-ml/ML_OPS.md b/spot-ml/ML_OPS.md
index 3251077..846cf35 100644
--- a/spot-ml/ML_OPS.md
+++ b/spot-ml/ML_OPS.md
@@ -2,7 +2,7 @@
ml_ops.sh is a helper script for invoking the suspicious connects analysis in the common case of its use in Spot.
-It loads environment variables from `/etc/spot.conf`, parses some arguments from the CLI, invokes the Suspicous Connects analysis from
+It loads environment variables from `/etc/spot.conf`, parses some arguments from the CLI, invokes the Suspicious Connects analysis from
the spot-ml jar stored in `target/scala-2.10/spot-ml-assembly-1.1.jar`.
@@ -12,7 +12,7 @@ The ml_ops script is meant to be used as part of a full spot deployment. Data is
Spot ingest and in a schema extending the [schema used by the suspicious connects analyses](SUSPICIOUS_CONNECTS_SCHEMA.md).
To run a suspicious connects analysis, execute the `ml_ops.sh` from the directory where the spot-ml jar is
-stored in the relativae path `target/scala-2.10/spot-ml-assembly-1.1.jar`
+stored in the relative path `target/scala-2.10/spot-ml-assembly-1.1.jar`
```
./ml_ops.sh YYYMMDD <type> <suspicion threshold> <max results>
@@ -25,7 +25,7 @@ stored in the relativae path `target/scala-2.10/spot-ml-assembly-1.1.jar`
* **suspicion threshold** is an optional floating-point value in the range [0,1]. All records that receive a score strictly greater than
the threshold are not reported. It defaults to 1 (no filter).
-* **max results** is an optional integer value value greater than or equal to 0. If provided, results are ordered by score from least to greatest and
+* **max results** is an optional integer value greater than or equal to 0. If provided, results are ordered by score from least to greatest and
the _max results_ smallest are returned in an ascending list. If not provided, all results with a score below the given threshold are returned in an ascending list.
## Example Usage
@@ -65,7 +65,7 @@ The ml_ops.sh script takes its values for the following parameters from the /etc
* **USER_DOMAIN** The domain name for the network being analyzed. Used to denote "internal" URLs during proxy and dns analyses.
* **TOPIC_COUNT** Number of topics used for the topic modelling at the heart of the Suspicious Connects anomaly detection. Roughly, the analysis attempts to generate TOPIC_COUNT many profiles of common traffic in the cluster.
* **DUPFACTOR** Used to downgrade the threat level of records similar to those marked as non-threatening by the feedback function of Spot UI. DUPFACTOR inflate the frequency of such records to make them appear less anomalous. A DUPFACTOR of 1 has no effect, and a DUPFACTOR of 1000 increases the frequency of the connection's pattern by a factor of 1000, increasing its estimated probability accordingly.
-* **TOL** The default value for the _suspicion threshold_ described above. In particular: If no third argument is provided to ml_ops.sh, the suspicion threshold is filled in with the TOL value from /etc/spot.conf. If a third argument is provided to ml_ops.sh, that is the supicion threshold used.
+* **TOL** The default value for the _suspicion threshold_ described above. In particular: If no third argument is provided to ml_ops.sh, the suspicion threshold is filled in with the TOL value from /etc/spot.conf. If a third argument is provided to ml_ops.sh, that is the suspicion threshold used.
* **LDA_OPTIMIZER** The LDA implementation to use. Set equal to "em" to execute LDA using EMLDAOptimizer or "online" to
use OnlineLDAOptimizer. See [LDA Spark documentation for more information.](https://spark.apache.org/docs/2.1.0/mllib-clustering.html#latent-dirichlet-allocation-lda)
* **LDA_ALPHA** Document concentration. See [LDA Spark documentation for more information.](https://spark.apache.org/docs/2.1.0/mllib-clustering.html#latent-dirichlet-allocation-lda)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/README.md
----------------------------------------------------------------------
diff --git a/spot-ml/README.md b/spot-ml/README.md
index 876ee77..42c8225 100644
--- a/spot-ml/README.md
+++ b/spot-ml/README.md
@@ -50,13 +50,28 @@ For example:
./ml_ops.sh 19731231 flow 1e-20 200
```
-You should have a list of the 200 most suspiciouc flow events from
+You should have a list of the 200 most suspicious flow events from
$HPATH/flow/scored_results/YYYYMMDD/scores/flow_results.csv
It is a csv file in which network events annotated with estimated probabilities and sorted in ascending order.
+## User Feedback
+
+The spot front end allows users to mark individual logged events as high, medium or low risk.
+
+The risk score is stored as a 1 for high risk, 2 for medium risk and 3 for low risk.
+
+At present, the scores of events similar to low risk items are inflated by the model, and nothing (at present) changes events flagged medium or high risk.
+
+
+This information is stored in a tab-separated text file stored on HDFS at:
+
+
+ /user/<user_name>/<data source>/scored_results/<date>/feedback/ml_feedback.csv
+
+
## Licensing
spot-ml is licensed under Apache Version 2.0
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/SPOT-ML-JAR.md
----------------------------------------------------------------------
diff --git a/spot-ml/SPOT-ML-JAR.md b/spot-ml/SPOT-ML-JAR.md
index 1171812..698a9b7 100644
--- a/spot-ml/SPOT-ML-JAR.md
+++ b/spot-ml/SPOT-ML-JAR.md
@@ -13,12 +13,12 @@ it is by submitting this class to Spark that the Suspicious Connects analyses ar
* **input** Path to data on HDFS. Data is expected to be stored in parquet with schema consistent with
[schema used by the suspicious connects analyses](SUSPICIOUS_CONNECTS_SCHEMA.md).
* **feedback** Local path of file containing feedback scores.
-* **dupfactor** Duplication factor controlling how to downrate non-threatening events from the feedback file.
+* **dupfactor** Duplication factor controlling how to down rate non-threatening events from the feedback file.
* **ldatopiccount** Number of topics in the topic model.
* **userdomain** The user domain of the network being analyzed.
* **scored** The HDFS path where results will be stored.
* **threshold** Threshold for determination of anomalies. Records with scores above the threshold will not be returned.
-* **maxresults** Maximum number of record to return. If -1, all records are returned. Results are filtered by the threshold and sorted and the most suspicous (lowest score) records are returned first.
+* **maxresults** Maximum number of record to return. If -1, all records are returned. Results are filtered by the threshold and sorted and the most suspicious (lowest score) records are returned first.
* **delimiter** Separation character used for CSVs containing most suspicious results.
* **prgseed** Seed for the pseudorandom generator used in topic modelling.
* **ldamaxiteration** Maximum number of iterations to execute the LDA topic modelling procedure.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/SUSPICIOUS_CONNECTS_SCHEMA.md
----------------------------------------------------------------------
diff --git a/spot-ml/SUSPICIOUS_CONNECTS_SCHEMA.md b/spot-ml/SUSPICIOUS_CONNECTS_SCHEMA.md
index 4afc6f2..7d0d5e2 100644
--- a/spot-ml/SUSPICIOUS_CONNECTS_SCHEMA.md
+++ b/spot-ml/SUSPICIOUS_CONNECTS_SCHEMA.md
@@ -2,7 +2,7 @@
-Three kinds of network logs can be analyzed by the Spot Suspicious Connects analyses: Netflow records, DNS queriess, and proxy logs.
+Three kinds of network logs can be analyzed by the Spot Suspicious Connects analyses: Netflow records, DNS queries, and proxy logs.
Data from these logs is expected to have been stored in parquet files on HDFS with column structure extending the following schema.
@@ -35,7 +35,7 @@ Data from these logs is expected to have been stored in parquet files on HDFS wi
- p_date: Date of the query, String.
- p_time: Time of the query, String. (FORMAT?)
- clientip: IP address of client making proxy request, String.
-- host: Host of reuest, String.
+- host: Host of request, String.
- reqmethod: Request method, String.
- useragent: User agent, String.
- resconttype: Response content type, String.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/build.sbt
----------------------------------------------------------------------
diff --git a/spot-ml/build.sbt b/spot-ml/build.sbt
index 0ac6e60..5ffc520 100644
--- a/spot-ml/build.sbt
+++ b/spot-ml/build.sbt
@@ -53,4 +53,20 @@ mergeStrategy in assembly <<= (mergeStrategy in assembly) { (old) => {
}
// super important with multiple tests running spark Contexts
-parallelExecution in Test := false
\ No newline at end of file
+parallelExecution in Test := false
+
+lazy val getTop1MFileFromAlexa = taskKey[Seq[File]]("Download Amazon/alexa-static top-1m.csv file for DNS and Proxy")
+
+getTop1MFileFromAlexa := {
+ if (!java.nio.file.Files.exists(new File("top-1m/top-1m.csv").toPath())) {
+ val location = url("http://s3.amazonaws.com/alexa-static/top-1m.csv.zip")
+ println("Getting top-1m.csv from Amazon/alexa-static")
+ IO.unzipURL(location, new File("top-1m")).toSeq
+ }
+ else {
+ println("File top-1m.csv already provided")
+ Seq.empty[File]
+ }
+}
+
+resourceGenerators in Compile <+= getTop1MFileFromAlexa
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/ml_ops.sh
----------------------------------------------------------------------
diff --git a/spot-ml/ml_ops.sh b/spot-ml/ml_ops.sh
index 4536676..dd00bbc 100755
--- a/spot-ml/ml_ops.sh
+++ b/spot-ml/ml_ops.sh
@@ -70,15 +70,10 @@ else
USER_DOMAIN_CMD=''
fi
-FEEDBACK_PATH=${LPATH}/${DSOURCE}_scores.csv
+FEEDBACK_PATH=${HPATH}/feedback/ml_feedback.csv
HDFS_SCORED_CONNECTS=${HPATH}/scores
-LDA_OUTPUT_DIR=${DSOURCE}/${FDATE}
-
-mkdir -p ${LPATH}
-rm -f ${LPATH}/*.{dat,beta,gamma,other,pkl} # protect the flow_scores.csv file
-
hdfs dfs -rm -R -f ${HDFS_SCORED_CONNECTS}
time spark-submit --class "org.apache.spot.SuspiciousConnects" \
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/ml_test.sh
----------------------------------------------------------------------
diff --git a/spot-ml/ml_test.sh b/spot-ml/ml_test.sh
index 9a064e6..3036c93 100755
--- a/spot-ml/ml_test.sh
+++ b/spot-ml/ml_test.sh
@@ -31,7 +31,6 @@ source /etc/spot.conf
TOL=1.1
MAXRESULTS=20
-LPATH=${LUSER}/ml/${DSOURCE}/test
HPATH=${HUSER}/${DSOURCE}/test/scored_results
# prepare parameters pipeline stages
@@ -43,13 +42,10 @@ else
USER_DOMAIN_CMD=''
fi
-FEEDBACK_PATH=${LPATH}/${DSOURCE}_scores.csv
+FEEDBACK_PATH=${HPATH}/feedback/ml_feedback.csv
HDFS_SCORED_CONNECTS=${HPATH}/scores
-mkdir -p ${LPATH}
-rm -f ${LPATH}/*.{dat,beta,gamma,other,pkl} # protect the flow_scores.csv file
-
hdfs dfs -rm -R -f ${HDFS_SCORED_CONNECTS}
time spark-submit --class "org.apache.spot.SuspiciousConnects" \
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/project/build.properties
----------------------------------------------------------------------
diff --git a/spot-ml/project/build.properties b/spot-ml/project/build.properties
new file mode 100644
index 0000000..b9b73f4
--- /dev/null
+++ b/spot-ml/project/build.properties
@@ -0,0 +1 @@
+sbt.version = 0.13.11
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/project/plugins.sbt
----------------------------------------------------------------------
diff --git a/spot-ml/project/plugins.sbt b/spot-ml/project/plugins.sbt
index 1f4b96a..55de9d9 100644
--- a/spot-ml/project/plugins.sbt
+++ b/spot-ml/project/plugins.sbt
@@ -1 +1,18 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.9.1")
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnects.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnects.scala b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnects.scala
index f3f122e..b0e55ba 100644
--- a/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnects.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/SuspiciousConnects.scala
@@ -71,12 +71,12 @@ object SuspiciousConnects {
}
val results: Option[SuspiciousConnectsAnalysisResults] = analysis match {
- case "flow" => Some(FlowSuspiciousConnectsAnalysis.run(config, sparkSession, logger,
- inputDataFrame))
- case "dns" => Some(DNSSuspiciousConnectsAnalysis.run(config, sparkSession, logger,
- inputDataFrame))
- case "proxy" => Some(ProxySuspiciousConnectsAnalysis.run(config, sparkSession, logger,
- inputDataFrame))
+ case "flow" => FlowSuspiciousConnectsAnalysis.run(config, sparkSession, logger,
+ inputDataFrame)
+ case "dns" => DNSSuspiciousConnectsAnalysis.run(config, sparkSession, logger,
+ inputDataFrame)
+ case "proxy" => ProxySuspiciousConnectsAnalysis.run(config, sparkSession, logger,
+ inputDataFrame)
case _ => None
}
@@ -103,7 +103,11 @@ object SuspiciousConnects {
InvalidDataHandler.showAndSaveInvalidRecords(invalidRecords, config.hdfsScoredConnect, logger)
}
- case None => logger.error("Unsupported (or misspelled) analysis: " + analysis)
+ case None => logger.error(s"Something went wrong while trying to run Suspicious Connects Analysis")
+ logger.error(s"Is the value of the analysis parameter (provided: $analysis) any of the valid analysis " +
+ s"types flow/dns/proxy?")
+ logger.error("If analysis type is correct please check for other errors like schema not matching or " +
+ "bad spark parameters")
}
sparkSession.stop()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysis.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysis.scala b/spot-ml/src/main/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysis.scala
index a1ec1ac..acf8fc6 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysis.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysis.scala
@@ -26,7 +26,8 @@ import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.dns.DNSSchema._
import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
import org.apache.spot.proxy.ProxySchema.Score
-import org.apache.spot.utilities.data.validation.{InvalidDataHandler => dataValidation}
+import org.apache.spot.utilities.data.validation.InputSchema.InputSchemaValidationResponse
+import org.apache.spot.utilities.data.validation.{InputSchema, InvalidDataHandler => dataValidation}
/**
* The suspicious connections analysis of DNS log data develops a probabilistic model the DNS queries
@@ -62,38 +63,45 @@ object DNSSuspiciousConnectsAnalysis {
* @param logger
*/
def run(config: SuspiciousConnectsConfig, sparkSession: SparkSession, logger: Logger,
- inputDNSRecords: DataFrame): SuspiciousConnectsAnalysisResults = {
+ inputDNSRecords: DataFrame): Option[SuspiciousConnectsAnalysisResults] = {
logger.info("Starting DNS suspicious connects analysis.")
- val dnsRecords = filterRecords(inputDNSRecords)
- .select(InSchema: _*)
- .na.fill(DefaultQueryClass, Seq(QueryClass))
- .na.fill(DefaultQueryType, Seq(QueryType))
- .na.fill(DefaultQueryResponseCode, Seq(QueryResponseCode))
+ logger.info("Validating schema...")
+ val InputSchemaValidationResponse(isValid, errorMessages) = validateSchema(inputDNSRecords)
- logger.info("Fitting probabilistic model to data")
- val model =
- DNSSuspiciousConnectsModel.trainModel(sparkSession, logger, config, dnsRecords)
+ if (!isValid) {
+ errorMessages.foreach(logger.error(_))
+ None
+ } else {
+ val dnsRecords = filterRecords(inputDNSRecords)
+ .select(InSchema: _*)
+ .na.fill(DefaultQueryClass, Seq(QueryClass))
+ .na.fill(DefaultQueryType, Seq(QueryType))
+ .na.fill(DefaultQueryResponseCode, Seq(QueryResponseCode))
- logger.info("Identifying outliers")
- val scoredDNSRecords = model.score(sparkSession, dnsRecords, config.userDomain, config.precisionUtility)
+ logger.info("Fitting probabilistic model to data")
+ val model =
+ DNSSuspiciousConnectsModel.trainModel(sparkSession, logger, config, dnsRecords)
- val filteredScored = filterScoredRecords(scoredDNSRecords, config.threshold)
+ logger.info("Identifying outliers")
+ val scoredDNSRecords = model.score(sparkSession, dnsRecords, config.userDomain, config.precisionUtility)
- val orderedDNSRecords = filteredScored.orderBy(Score)
+ val filteredScored = filterScoredRecords(scoredDNSRecords, config.threshold)
- val mostSuspiciousDNSRecords =
- if (config.maxResults > 0) orderedDNSRecords.limit(config.maxResults)
- else orderedDNSRecords
+ val orderedDNSRecords = filteredScored.orderBy(Score)
- val outputDNSRecords = mostSuspiciousDNSRecords.select(OutSchema: _*)
+ val mostSuspiciousDNSRecords =
+ if (config.maxResults > 0) orderedDNSRecords.limit(config.maxResults)
+ else orderedDNSRecords
- val invalidDNSRecords = filterInvalidRecords(inputDNSRecords).select(InSchema: _*)
+ val outputDNSRecords = mostSuspiciousDNSRecords.select(OutSchema: _*)
- SuspiciousConnectsAnalysisResults(outputDNSRecords, invalidDNSRecords)
+ val invalidDNSRecords = filterInvalidRecords(inputDNSRecords).select(InSchema: _*)
+ Option(SuspiciousConnectsAnalysisResults(outputDNSRecords, invalidDNSRecords))
+ }
}
@@ -158,7 +166,7 @@ object DNSSuspiciousConnectsAnalysis {
/**
*
* @param scoredDNSRecords scored DNS records.
- * @param threshold score tolerance.
+ * @param threshold score tolerance.
* @return
*/
def filterScoredRecords(scoredDNSRecords: DataFrame, threshold: Double): DataFrame = {
@@ -170,4 +178,16 @@ object DNSSuspiciousConnectsAnalysis {
scoredDNSRecords.filter(filteredDNSRecordsFilter)
}
+ /**
+ * Validates incoming data matches required schema for model training and scoring.
+ *
+ * @param inputDNSRecords incoming data frame
+ * @return
+ */
+ def validateSchema(inputDNSRecords: DataFrame): InputSchemaValidationResponse = {
+
+ InputSchema.validate(inputDNSRecords.schema, DNSSuspiciousConnectsModel.ModelSchema)
+
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSFeedback.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSFeedback.scala b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSFeedback.scala
index 227b836..d218f77 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSFeedback.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSFeedback.scala
@@ -20,8 +20,7 @@ package org.apache.spot.dns.model
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spot.dns.model.DNSSuspiciousConnectsModel.{ModelSchema, modelColumns}
-
-import scala.io.Source
+import org.apache.spot.utilities.data.InputOutputDataHandler.getFeedbackRDD
/**
* Routines for ingesting the feedback file provided by the operational analytics layer.
@@ -42,14 +41,9 @@ object DNSFeedback {
duplicationFactor: Int): DataFrame = {
- if (new java.io.File(feedbackFile).exists) {
-
- /*
- feedback file is a tab-separated file with a single header line.
- */
+ val feedback: RDD[String] = getFeedbackRDD(sparkSession, feedbackFile)
- val lines = Source.fromFile(feedbackFile).getLines().toArray.drop(1)
- val feedback: RDD[String] = sparkSession.sparkContext.parallelize(lines)
+ if (!feedback.isEmpty()) {
/*
The columns and their entries are as follows:
@@ -93,7 +87,7 @@ object DNSFeedback {
row(DnsQryTypeIndex).toInt,
row(DnsQryRcodeIndex).toInt)))
.flatMap(row => List.fill(duplicationFactor)(row)), ModelSchema)
- .select(modelColumns:_*)
+ .select(modelColumns: _*)
} else {
sparkSession.createDataFrame(sparkSession.sparkContext.emptyRDD[Row], ModelSchema)
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
index 3089bc8..dfcb543 100644
--- a/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/dns/model/DNSSuspiciousConnectsModel.scala
@@ -159,12 +159,8 @@ object DNSSuspiciousConnectsModel {
val topDomainsBC = sparkSession.sparkContext.broadcast(TopDomains.TopDomains)
val userDomain = config.userDomain
- val domainStatsRecords = createDomainStatsDF(sparkSession, countryCodesBC, topDomainsBC, userDomain, totalRecords)
-
// simplify DNS log entries into "words"
-
val dnsWordCreator = new DNSWordCreation(topDomainsBC, userDomain)
-
val dataWithWord = totalRecords.withColumn(Word, dnsWordCreator.wordCreationUDF(modelColumns: _*))
import sparkSession.implicits._
@@ -195,30 +191,6 @@ object DNSSuspiciousConnectsModel {
}
- /**
- * Add domain statistics fields to a data frame.
- *
- * @param sparkSession Spark Session
- * @param countryCodesBC Broadcast of the country codes set.
- * @param topDomainsBC Broadcast of the most-popular domains set.
- * @param userDomain Domain associated to network data (ex: 'intel')
- * @param inDF Incoming dataframe. Schema is expected to provide the field [[QueryName]]
- * @return A new dataframe with the new columns added. The new columns have the schema [[DomainStatsSchema]]
- */
-
- def createDomainStatsDF(sparkSession: SparkSession,
- countryCodesBC: Broadcast[Set[String]],
- topDomainsBC: Broadcast[Set[String]],
- userDomain: String,
- inDF: DataFrame): DataFrame = {
-
- val queryNameIndex = inDF.schema.fieldNames.indexOf(QueryName)
-
- val domainStatsRDD: RDD[Row] = inDF.rdd.map(row =>
- Row.fromTuple(createTempFields(countryCodesBC, topDomainsBC, userDomain, row.getString(queryNameIndex))))
-
- sparkSession.createDataFrame(domainStatsRDD, DomainStatsSchema)
- }
/**
*
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
index b6fe791..b2d1304 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysis.scala
@@ -25,7 +25,8 @@ import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.netflow.FlowSchema._
import org.apache.spot.netflow.model.FlowSuspiciousConnectsModel
-import org.apache.spot.utilities.data.validation.{InvalidDataHandler => dataValidation}
+import org.apache.spot.utilities.data.validation.InputSchema.InputSchemaValidationResponse
+import org.apache.spot.utilities.data.validation.{InputSchema, InvalidDataHandler => dataValidation}
/**
@@ -83,32 +84,40 @@ object FlowSuspiciousConnectsAnalysis {
* @return
*/
def run(config: SuspiciousConnectsConfig, sparkSession: SparkSession, logger: Logger,
- inputFlowRecords: DataFrame): SuspiciousConnectsAnalysisResults = {
+ inputFlowRecords: DataFrame): Option[SuspiciousConnectsAnalysisResults] = {
logger.info("Starting flow suspicious connects analysis.")
- val flowRecords = filterRecords(inputFlowRecords).select(InSchema: _*)
+ logger.info("Validating schema...")
+ val InputSchemaValidationResponse(isValid, errorMessages) = validateSchema(inputFlowRecords)
- logger.info("Fitting probabilistic model to data")
- val model =
- FlowSuspiciousConnectsModel.trainModel(sparkSession, logger, config, flowRecords)
+ if (!isValid) {
+ errorMessages.foreach(logger.error(_))
+ None
- logger.info("Identifying outliers")
- val scoredFlowRecords = model.score(sparkSession, flowRecords, config.precisionUtility)
+ } else {
+ val flowRecords = filterRecords(inputFlowRecords).select(InSchema: _*)
- val filteredScored = filterScoredRecords(scoredFlowRecords, config.threshold)
+ logger.info("Fitting probabilistic model to data")
+ val model =
+ FlowSuspiciousConnectsModel.trainModel(sparkSession, logger, config, flowRecords)
- val orderedFlowRecords = filteredScored.orderBy(Score)
+ logger.info("Identifying outliers")
+ val scoredFlowRecords = model.score(sparkSession, flowRecords, config.precisionUtility)
- val mostSuspiciousFlowRecords =
- if (config.maxResults > 0) orderedFlowRecords.limit(config.maxResults) else orderedFlowRecords
+ val filteredScored = filterScoredRecords(scoredFlowRecords, config.threshold)
- val outputFlowRecords = mostSuspiciousFlowRecords.select(OutSchema: _*)
+ val orderedFlowRecords = filteredScored.orderBy(Score)
- val invalidFlowRecords = filterInvalidRecords(inputFlowRecords).select(InSchema: _*)
+ val mostSuspiciousFlowRecords =
+ if (config.maxResults > 0) orderedFlowRecords.limit(config.maxResults) else orderedFlowRecords
- SuspiciousConnectsAnalysisResults(outputFlowRecords, invalidFlowRecords)
+ val outputFlowRecords = mostSuspiciousFlowRecords.select(OutSchema: _*)
+ val invalidFlowRecords = filterInvalidRecords(inputFlowRecords).select(InSchema: _*)
+
+ Option(SuspiciousConnectsAnalysisResults(outputFlowRecords, invalidFlowRecords))
+ }
}
/**
@@ -177,4 +186,16 @@ object FlowSuspiciousConnectsAnalysis {
scoredFlowRecords.filter(filteredFlowRecordsFilter)
}
+ /**
+ * Validates incoming data matches required schema for model training and scoring.
+ *
+ * @param inputFlowRecords incoming data frame
+ * @return
+ */
+ def validateSchema(inputFlowRecords: DataFrame): InputSchemaValidationResponse = {
+
+ InputSchema.validate(inputFlowRecords.schema, FlowSuspiciousConnectsModel.ModelSchema)
+
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowFeedback.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowFeedback.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowFeedback.scala
index 3bbfbe1..832d670 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowFeedback.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowFeedback.scala
@@ -20,8 +20,7 @@ package org.apache.spot.netflow.model
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spot.netflow.model.FlowSuspiciousConnectsModel._
-
-import scala.io.Source
+import org.apache.spot.utilities.data.InputOutputDataHandler.getFeedbackRDD
/**
* Routines for ingesting the feedback file provided by the operational analytics layer.
@@ -42,16 +41,9 @@ object FlowFeedback {
feedbackFile: String,
duplicationFactor: Int): DataFrame = {
+ val feedback: RDD[String] = getFeedbackRDD(sparkSession, feedbackFile)
- if (new java.io.File(feedbackFile).exists) {
-
- /*
- feedback file is a tab-separated file with a single header line.
- */
-
- val lines = Source.fromFile(feedbackFile).getLines().toArray.drop(1)
- val feedback: RDD[String] = sparkSession.sparkContext.parallelize(lines)
-
+ if (!feedback.isEmpty()) {
/*
flow_scores.csv - feedback file structure
0 sev
@@ -75,9 +67,6 @@ object FlowFeedback {
val DestinationPortIndex = 5
val IpktIndex = 6
val IbytIndex = 7
- val HourIndex = 20
- val MinuteIndex = 21
- val SecondIndex = 22
sparkSession.createDataFrame(feedback.map(_.split("\t"))
.filter(row => row(ScoreIndex).trim.toInt == 3)
@@ -87,8 +76,8 @@ object FlowFeedback {
row(TimeStartIndex).split(" ")(1).split(":")(2).trim.toInt, // second
row(SourceIpIndex),
row(DestinationIpIndex),
- row(SourcePortIndex),
- row(DestinationPortIndex),
+ row(SourcePortIndex).trim.toInt,
+ row(DestinationPortIndex).trim.toInt,
row(IpktIndex).trim.toLong,
row(IbytIndex).trim.toLong)))
.flatMap(row => List.fill(duplicationFactor)(row)), ModelSchema)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
index c29260a..6be11e1 100644
--- a/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModel.scala
@@ -141,7 +141,7 @@ object FlowSuspiciousConnectsModel {
val selectedRecords = inputRecords.select(ModelColumns: _*)
- val totalRecords = selectedRecords.unionAll(FlowFeedback.loadFeedbackDF(sparkSession,
+ val totalRecords = selectedRecords.union(FlowFeedback.loadFeedbackDF(sparkSession,
config.feedbackFile,
config.duplicationFactor))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyFeedback.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyFeedback.scala b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyFeedback.scala
index 06acac7..c2bb3e1 100644
--- a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyFeedback.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxyFeedback.scala
@@ -21,8 +21,7 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spot.proxy.ProxySchema._
-
-import scala.io.Source
+import org.apache.spot.utilities.data.InputOutputDataHandler.getFeedbackRDD
object ProxyFeedback {
@@ -41,17 +40,19 @@ object ProxyFeedback {
val feedbackSchema = StructType(
- List(StructField(Date, StringType, nullable= true),
- StructField(Time, StringType, nullable= true),
- StructField(ClientIP, StringType, nullable= true),
- StructField(Host, StringType, nullable= true),
- StructField(ReqMethod, StringType, nullable= true),
- StructField(UserAgent, StringType, nullable= true),
- StructField(ResponseContentType, StringType, nullable= true),
- StructField(RespCode, StringType, nullable= true),
- StructField(FullURI, StringType, nullable= true)))
+ List(StructField(Date, StringType, nullable = true),
+ StructField(Time, StringType, nullable = true),
+ StructField(ClientIP, StringType, nullable = true),
+ StructField(Host, StringType, nullable = true),
+ StructField(ReqMethod, StringType, nullable = true),
+ StructField(UserAgent, StringType, nullable = true),
+ StructField(ResponseContentType, StringType, nullable = true),
+ StructField(RespCode, StringType, nullable = true),
+ StructField(FullURI, StringType, nullable = true)))
+
+ val feedback: RDD[String] = getFeedbackRDD(sparkSession, feedbackFile)
- if (new java.io.File(feedbackFile).exists) {
+ if (!feedback.isEmpty()) {
val dateIndex = 0
val timeIndex = 1
@@ -65,9 +66,6 @@ object ProxyFeedback {
val fullURISeverityIndex = 22
- val lines = Source.fromFile(feedbackFile).getLines().toArray.drop(1)
- val feedback: RDD[String] = sparkSession.sparkContext.parallelize(lines)
-
sparkSession.createDataFrame(feedback.map(_.split("\t"))
.filter(row => row(fullURISeverityIndex).trim.toInt == 3)
.map(row => Row.fromSeq(List(row(dateIndex),
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysis.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysis.scala b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysis.scala
index 90ac063..ea0b90a 100644
--- a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysis.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysis.scala
@@ -24,7 +24,8 @@ import org.apache.spark.sql.{DataFrame, SparkSession}
import org.apache.spot.SuspiciousConnects.SuspiciousConnectsAnalysisResults
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.proxy.ProxySchema._
-import org.apache.spot.utilities.data.validation.{InvalidDataHandler => dataValidation}
+import org.apache.spot.utilities.data.validation.InputSchema.InputSchemaValidationResponse
+import org.apache.spot.utilities.data.validation.{InputSchema, InvalidDataHandler => dataValidation}
/**
* Run suspicious connections analysis on proxy data.
@@ -84,36 +85,45 @@ object ProxySuspiciousConnectsAnalysis {
* @param logger Logs execution progress, information and errors for user.
*/
def run(config: SuspiciousConnectsConfig, sparkSession: SparkSession, logger: Logger,
- inputProxyRecords: DataFrame): SuspiciousConnectsAnalysisResults = {
+ inputProxyRecords: DataFrame): Option[SuspiciousConnectsAnalysisResults] = {
logger.info("Starting proxy suspicious connects analysis.")
- val proxyRecords = filterRecords(inputProxyRecords)
- .select(InSchema: _*)
- .na.fill(DefaultUserAgent, Seq(UserAgent))
- .na.fill(DefaultResponseContentType, Seq(ResponseContentType))
+ logger.info("Validating schema...")
+ val InputSchemaValidationResponse(isValid, errorMessages) = validateSchema(inputProxyRecords)
- logger.info("Fitting probabilistic model to data")
- val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, config, proxyRecords)
+ if (!isValid) {
+ errorMessages.foreach(logger.error(_))
- logger.info("Identifying outliers")
- val scoredProxyRecords = model.score(sparkSession, proxyRecords, config.precisionUtility)
+ None
+ } else {
+ val proxyRecords = filterRecords(inputProxyRecords)
+ .select(InSchema: _*)
+ .na.fill(DefaultUserAgent, Seq(UserAgent))
+ .na.fill(DefaultResponseContentType, Seq(ResponseContentType))
- // take the maxResults least probable events of probability below the threshold and sort
+ logger.info("Fitting probabilistic model to data")
+ val model = ProxySuspiciousConnectsModel.trainModel(sparkSession, logger, config, proxyRecords)
- val filteredScored = filterScoredRecords(scoredProxyRecords, config.threshold)
+ logger.info("Identifying outliers")
+ val scoredProxyRecords = model.score(sparkSession, proxyRecords, config.precisionUtility)
- val orderedProxyRecords = filteredScored.orderBy(Score)
+ // take the maxResults least probable events of probability below the threshold and sort
- val mostSuspiciousProxyRecords =
- if (config.maxResults > 0) orderedProxyRecords.limit(config.maxResults)
- else orderedProxyRecords
+ val filteredScored = filterScoredRecords(scoredProxyRecords, config.threshold)
- val outputProxyRecords = mostSuspiciousProxyRecords.select(OutSchema: _*)
+ val orderedProxyRecords = filteredScored.orderBy(Score)
- val invalidProxyRecords = filterInvalidRecords(inputProxyRecords).select(InSchema: _*)
+ val mostSuspiciousProxyRecords =
+ if (config.maxResults > 0) orderedProxyRecords.limit(config.maxResults)
+ else orderedProxyRecords
- SuspiciousConnectsAnalysisResults(outputProxyRecords, invalidProxyRecords)
+ val outputProxyRecords = mostSuspiciousProxyRecords.select(OutSchema: _*)
+
+ val invalidProxyRecords = filterInvalidRecords(inputProxyRecords).select(InSchema: _*)
+
+ Option(SuspiciousConnectsAnalysisResults(outputProxyRecords, invalidProxyRecords))
+ }
}
@@ -165,4 +175,16 @@ object ProxySuspiciousConnectsAnalysis {
scoredProxyRecords.filter(filteredProxyRecordsFilter)
}
+ /**
+ * Validates incoming data matches required schema for model training and scoring.
+ *
+ * @param inputProxyRecords incoming data frame
+ * @return
+ */
+ def validateSchema(inputProxyRecords: DataFrame): InputSchemaValidationResponse = {
+
+ InputSchema.validate(inputProxyRecords.schema, ProxySuspiciousConnectsModel.ModelSchema)
+
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
index 0cf006b..3ef60af 100644
--- a/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/proxy/ProxySuspiciousConnectsModel.scala
@@ -21,6 +21,7 @@ import org.apache.log4j.Logger
import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.RDD
import org.apache.spark.sql.functions._
+import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.{DataFrame, Row, SparkSession}
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
import org.apache.spot.SuspiciousConnectsScoreFunction
@@ -100,6 +101,18 @@ object ProxySuspiciousConnectsModel {
3.0, 3.3, 3.6, 3.9, 4.2,
4.5, 4.8, 5.1, 5.4, Double.PositiveInfinity)
+ val ModelSchema = StructType(List(DateField,
+ TimeField,
+ ClientIPField,
+ HostField,
+ ReqMethodField,
+ UserAgentField,
+ ResponseContentTypeField,
+ RespCodeField,
+ FullURIField))
+
+ val ModelColumns = ModelSchema.fieldNames.toList.map(col)
+
/**
* Factory for ProxySuspiciousConnectsModel.
* Trains the model from the incoming DataFrame using the specified number of topics
@@ -121,9 +134,8 @@ object ProxySuspiciousConnectsModel {
val selectedRecords =
- inputRecords.select(Date, Time, ClientIP, Host, ReqMethod, UserAgent, ResponseContentType, RespCode, FullURI)
- .unionAll(ProxyFeedback.loadFeedbackDF(sparkSession, config.feedbackFile, config.duplicationFactor))
-
+ inputRecords.select(ModelColumns: _*)
+ .union(ProxyFeedback.loadFeedbackDF(sparkSession, config.feedbackFile, config.duplicationFactor))
val agentToCount: Map[String, Long] =
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
index b9592ea..0183027 100644
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/TopDomains.scala
@@ -23,7 +23,8 @@ import scala.io.Source
* List of top domains used for DNS and Proxy analysis.
*/
object TopDomains {
- val alexaTop1MPath = "top-1m.csv"
+ // Auto-generated by SBT at compile time
+ val alexaTop1MPath = "top-1m/top-1m.csv"
val TopDomains: Set[String] = Source.fromFile(alexaTop1MPath).getLines.map(line => {
val parts = line.split(",")
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
index 481e389..2d5e27b 100644
--- a/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/data/InputOutputDataHandler.scala
@@ -17,10 +17,11 @@
package org.apache.spot.utilities.data
-import org.apache.hadoop.fs.{LocatedFileStatus, Path, RemoteIterator, FileUtil => fileUtil}
+import org.apache.hadoop.fs.{FileSystem, LocatedFileStatus, Path, RemoteIterator, FileUtil => fileUtil}
import org.apache.log4j.Logger
import org.apache.spark.sql.{DataFrame, SparkSession}
-
+import org.apache.spark.rdd.RDD
+import org.apache.spark.sql.{DataFrame, SQLContext}
/**
* Handles input and output data for every data set or pipep line implementation.
@@ -47,6 +48,34 @@ object InputOutputDataHandler {
/**
*
* @param sparkSession Spark Session.
+ * @param feedbackFile Feedback file location.
+ * @return new RDD[String] with feedback or empty if file does not exists.
+ */
+ def getFeedbackRDD(sparkSession: SparkSession, feedbackFile: String): RDD[String] = {
+
+ val hadoopConfiguration = sparkSession.sparkContext.hadoopConfiguration
+ val fs = FileSystem.get(hadoopConfiguration)
+
+ // We need to pass a default value "file" if fileName is "" to avoid error
+ // java.lang.IllegalArgumentException: Can not create a Path from an empty string
+ // when trying to create a new Path object with empty string.
+ val fileExists = fs.exists(new Path(if (feedbackFile == "") "file" else feedbackFile))
+
+ if (fileExists) {
+
+ // feedback file is a tab-separated file with a single header line. We need to remove the header
+ val lines = sparkSession.sparkContext.textFile(feedbackFile)
+ val header = lines.first()
+ lines.filter(line => line != header)
+
+ } else {
+ sparkSession.sparkContext.emptyRDD[String]
+ }
+ }
+
+ /**
+ *
+ * @param sparkSession Spark Session
* @param hdfsScoredConnect HDFS output folder. The location where results were saved; flow, dns or proxy.
* @param analysis Data type to analyze.
* @param logger Application Logger.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/main/scala/org/apache/spot/utilities/data/validation/InputSchema.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/main/scala/org/apache/spot/utilities/data/validation/InputSchema.scala b/spot-ml/src/main/scala/org/apache/spot/utilities/data/validation/InputSchema.scala
new file mode 100644
index 0000000..997d992
--- /dev/null
+++ b/spot-ml/src/main/scala/org/apache/spot/utilities/data/validation/InputSchema.scala
@@ -0,0 +1,51 @@
+package org.apache.spot.utilities.data.validation
+
+import org.apache.spark.sql.types.{DataType, StructType}
+
+import scala.collection.mutable.ListBuffer
+
+
+/**
+ * Input schema routines
+ */
+object InputSchema {
+
+ /**
+ * Validate the incoming data schema matches the schema required for model creation and scoring.
+ *
+ * @param inSchema incoming data frame
+ * @param expectedSchema schema expected by model training and scoring methods
+ * @return
+ */
+ def validate(inSchema: StructType, expectedSchema: StructType): InputSchemaValidationResponse = {
+ val response: ListBuffer[String] = ListBuffer("Schema not compatible:")
+
+ // reduce schema from struct field to only field name and type
+ val inSchemaMap: Map[String, DataType] = inSchema.map(field => (field.name -> field.dataType)).toMap
+
+ expectedSchema
+ .map(field => (field.name, field.dataType))
+ .foreach({ case (expectedFieldName: String, expectedDataType: DataType) => {
+
+ val inFieldDataType = inSchemaMap.getOrElse(expectedFieldName, None)
+
+ inFieldDataType match {
+ case None => response.append(s"Field $expectedFieldName is not present. $expectedFieldName is required for " +
+ s"model training and scoring.")
+ case inputDataType: DataType =>
+ if (inputDataType != expectedDataType)
+ response.append(s"Field $expectedFieldName type ${inputDataType.typeName} is not the expected type " +
+ s"${expectedDataType.typeName}")
+ }
+ }
+ })
+
+ response.length match {
+ case 1 => InputSchemaValidationResponse(isValid = true, Seq())
+ case _ => InputSchemaValidationResponse(isValid = false, response)
+ }
+ }
+
+ case class InputSchemaValidationResponse(final val isValid: Boolean, final val errorMessages: Seq[String])
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/resources/log4j.properties b/spot-ml/src/test/resources/log4j.properties
index feca830..36416f3 100644
--- a/spot-ml/src/test/resources/log4j.properties
+++ b/spot-ml/src/test/resources/log4j.properties
@@ -1,8 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
# Change this to set Spark log level
log4j.logger.org.apache.spark=ERROR
-
# Silence akka remoting
log4j.logger.Remoting=ERROR
-
# Ignore messages below warning level from Jetty, because it's a bit verbose
log4j.logger.org.eclipse.jetty=ERROR
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
index f61b5ed..aaf9539 100644
--- a/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/SuspiciousConnectsArgumentParserTest.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.spot
import org.apache.spot.SuspiciousConnectsArgumentParser.SuspiciousConnectsConfig
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
index 5ceccaa..5aafbb2 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsAnalysisTest.scala
@@ -39,6 +39,15 @@ case class DNSInput(frame_time: String,
dns_qry_type: Int,
dns_qry_rcode: Int)
+case class DNSInvalidSchema(frame_time: String,
+ unix_tstamp: Double,
+ frame_len: Int,
+ ip_dst: String,
+ dns_qry_name: String,
+ dns_qry_class: String,
+ dns_qry_type: String,
+ dns_qry_rcode: String)
+
class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with Matchers {
val emTestConfig = SuspiciousConnectsConfig(analysis = "dns",
@@ -101,6 +110,8 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
val SuspiciousConnectsAnalysisResults(scoredData, _) =
DNSSuspiciousConnectsAnalysis.run(emTestConfig, sparkSession, logger, data)
+ .getOrElse(SuspiciousConnectsAnalysisResults(sparkSession.sqlContext.emptyDataFrame,
+ sparkSession.sqlContext.emptyDataFrame))
val anomalyScore = scoredData.filter(scoredData(FrameLength) === 1).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(FrameLength) === 168).collect().map(_.getAs[Double](Score))
@@ -123,8 +134,10 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
val typicalRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 168, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val SuspiciousConnectsAnalysisResults(scoredData, _) = DNSSuspiciousConnectsAnalysis.run(onlineTestConfig,
- sparkSession, logger, data)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) =
+ DNSSuspiciousConnectsAnalysis.run(onlineTestConfig, sparkSession, logger, data)
+ .getOrElse(SuspiciousConnectsAnalysisResults(sparkSession.sqlContext.emptyDataFrame,
+ sparkSession.sqlContext.emptyDataFrame))
val anomalyScore = scoredData.filter(scoredData(FrameLength) === 1).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(FrameLength) === 168).collect().map(_.getAs[Double](Score))
@@ -284,6 +297,26 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
Math.abs(typicalScores(3) - 0.8d) should be <= 0.01d
}
+ it should "return None when input data frame doesn't matches expected schema" in {
+ val logger = LogManager.getLogger("SuspiciousConnectsAnalysis")
+ logger.setLevel(Level.WARN)
+
+ val anomalousRecord = DNSInvalidSchema("May 20 2016 02:10:25.970987000 PDT", 1463735425d, 1, "172.16.9.132",
+ "122.2o7.turner.com", "0x00000001", "null", "null")
+ val data = sparkSession.createDataFrame(Seq(anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord))
+
+ val result: Option[SuspiciousConnectsAnalysisResults] =
+ DNSSuspiciousConnectsAnalysis.run(emTestConfig, sparkSession, logger, data)
+
+ val none = result match {
+ case None => true
+ case Some(_) => false
+ }
+
+ none shouldBe true
+
+ }
+
"filterRecords" should "return data set without garbage" in {
val cleanedDNSRecords = DNSSuspiciousConnectsAnalysis.filterRecords(testDNSRecords.inputDNSRecordsDF)
@@ -309,6 +342,25 @@ class DNSSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with
scoredDNSRecords.count should be(2)
}
+ "validateSchema" should "return false when passing an invalid schema" in {
+ val anomalousRecord = DNSInvalidSchema("May 20 2016 02:10:25.970987000 PDT", 1463735425d, 1, "172.16.9.132",
+ "122.2o7.turner.com", "0x00000001", "null", "null")
+ val data = sparkSession.createDataFrame(Seq(anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord))
+
+ val results = DNSSuspiciousConnectsAnalysis.validateSchema(data)
+
+ results.isValid shouldBe false
+ }
+
+ it should "return true when passing a valid schema" in {
+ val typicalRecord = DNSInput("May 20 2016 02:10:25.970987000 PDT", 1463735425L, 168, "172.16.9.132", "122.2o7.turner.com", "0x00000001", 1, 0)
+ val data = sparkSession.createDataFrame(Seq(typicalRecord, typicalRecord, typicalRecord, typicalRecord))
+
+ val results = DNSSuspiciousConnectsAnalysis.validateSchema(data)
+
+ results.isValid shouldBe true
+ }
+
def testDNSRecords = new {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
index f5a2d79..77b98d7 100644
--- a/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/dns/DNSSuspiciousConnectsModelTest.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.spot.dns
import org.apache.spot.dns.model.DNSSuspiciousConnectsModel
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/netflow/FlowRecord.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowRecord.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowRecord.scala
index 88b84b3..19e51be 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowRecord.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowRecord.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.spot.netflow
/**
@@ -20,3 +37,21 @@ case class FlowRecord(treceived: String,
ibyt: Long,
opkt: Long,
obyt: Long)
+
+case class FlowInvalidRecord(treceived: String,
+ tryear: Int,
+ trmonth: Int,
+ trday: Int,
+ trhour: Int,
+ trminute: Int,
+ trsec: Int,
+ tdur: Int,
+ sip: String,
+ dip: String,
+ sport: Int,
+ dport: Int,
+ proto: String,
+ ipkt: String,
+ ibyt: String,
+ opkt: String,
+ obyt: String)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
index 9310390..f7bdb12 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/FlowSuspiciousConnectsAnalysisTest.scala
@@ -92,8 +92,10 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val SuspiciousConnectsAnalysisResults(scoredData, _) = FlowSuspiciousConnectsAnalysis.run(emTestConfig,
- sparkSession, logger, data)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) =
+ FlowSuspiciousConnectsAnalysis.run(emTestConfig, sparkSession, logger, data)
+ .getOrElse(SuspiciousConnectsAnalysisResults(sparkSession.sqlContext.emptyDataFrame,
+ sparkSession.sqlContext.emptyDataFrame))
val anomalyScore = scoredData.filter(scoredData(Hour) === 0).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Hour) === 13).collect().map(_.getAs[Double](Score))
@@ -124,8 +126,10 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val SuspiciousConnectsAnalysisResults(scoredData, _) = FlowSuspiciousConnectsAnalysis.run(onlineTestConfig,
- sparkSession, logger, data)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) =
+ FlowSuspiciousConnectsAnalysis.run(onlineTestConfig, sparkSession, logger, data)
+ .getOrElse(SuspiciousConnectsAnalysisResults(sparkSession.sqlContext.emptyDataFrame,
+ sparkSession.sqlContext.emptyDataFrame))
val anomalyScore = scoredData.filter(scoredData(Hour) === 0).first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Hour) === 13).collect().map(_.getAs[Double](Score))
@@ -239,6 +243,27 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
}
+ it should "return None when input data frame doesn't matches expected schema" in {
+ val logger = LogManager.getLogger("SuspiciousConnectsAnalysis")
+ logger.setLevel(Level.OFF)
+
+ val anomalousRecord = FlowInvalidRecord("2016-05-05 00:11:01", 2016, 5, 5, 0, 0, 1, 972, "172.16.0.129", "10.0" +
+ ".2.202", 1024, 80, "TCP", "39l", "12522l", "0l", "0l")
+
+ val data = sparkSession.createDataFrame(Seq(anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord))
+
+ val result =
+ FlowSuspiciousConnectsAnalysis.run(emTestConfig, sparkSession, logger, data)
+
+ val none = result match {
+ case None => true
+ case Some(_) => false
+ }
+
+ none shouldBe true
+
+ }
+
"filterRecords" should "return data set without garbage" in {
val cleanedFlowRecords = FlowSuspiciousConnectsAnalysis
@@ -267,6 +292,29 @@ class FlowSuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wit
scoredFlowRecords.count should be(2)
}
+ "validateSchema" should "return false when passing an invalid schema" in {
+ val anomalousRecord = FlowInvalidRecord("2016-05-05 00:11:01", 2016, 5, 5, 0, 0, 1, 972, "172.16.0.129", "10.0" +
+ ".2.202", 1024, 80, "TCP", "39l", "12522l", "0l", "0l")
+
+ val data = sparkSession.createDataFrame(Seq(anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord))
+
+ val results = FlowSuspiciousConnectsAnalysis.validateSchema(data)
+
+ results.isValid shouldBe false
+ }
+
+ it should "return true when passing a valid schema" in {
+ val typicalRecord = FlowRecord("2016-05-05 13:54:58", 2016, 5, 5, 13, 54, 58, 0.972f, "172.16.0.129", "10.0.2" +
+ ".202", 1024, 80, "TCP", 39, 12522, 0, 0)
+
+ val data = sparkSession.createDataFrame(Seq(typicalRecord, typicalRecord, typicalRecord, typicalRecord,
+ typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
+
+ val results = FlowSuspiciousConnectsAnalysis.validateSchema(data)
+
+ results.isValid shouldBe true
+ }
+
def testFlowRecords = new {
val inputFlowRecordsRDD = sparkSession.sparkContext.parallelize(wrapRefArray(Array(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModelTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModelTest.scala b/spot-ml/src/test/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModelTest.scala
index ac0ec12..618b614 100644
--- a/spot-ml/src/test/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModelTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/netflow/model/FlowSuspiciousConnectsModelTest.scala
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.spot.netflow.model
import org.apache.spark.sql.Row
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
----------------------------------------------------------------------
diff --git a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
index e7dfd04..d8e4061 100644
--- a/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
+++ b/spot-ml/src/test/scala/org/apache/spot/proxy/ProxySuspiciousConnectsAnalysisTest.scala
@@ -47,6 +47,26 @@ case class ProxyInput(p_date: String,
csbytes: Int,
fulluri: String)
+case class ProxyInvalidInput(p_date: String,
+ p_time: String,
+ clientip: String,
+ host: Double,
+ reqmethod: String,
+ useragent: String,
+ resconttype: Int,
+ duration: Int,
+ username: String,
+ webcat: String,
+ referer: String,
+ respcode: String,
+ uriport: Int,
+ uripath: String,
+ uriquery: String,
+ serverip: String,
+ scbytes: Int,
+ csbytes: Int,
+ fulluri: Double)
+
class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec with Matchers {
@@ -107,8 +127,10 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
val data = sparkSession.createDataFrame(Seq(anomalousRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord,
typicalRecord, typicalRecord, typicalRecord, typicalRecord, typicalRecord))
- val SuspiciousConnectsAnalysisResults(scoredData, _) = ProxySuspiciousConnectsAnalysis.run(emTestConfigProxy,
- sparkSession, logger, data)
+ val SuspiciousConnectsAnalysisResults(scoredData, _) =
+ ProxySuspiciousConnectsAnalysis.run(emTestConfigProxy, sparkSession, logger, data)
+ .getOrElse(SuspiciousConnectsAnalysisResults(sparkSession.sqlContext.emptyDataFrame,
+ sparkSession.sqlContext.emptyDataFrame))
val anomalyScore = scoredData.filter(scoredData(Host) === "intel.com").first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Host) === "maw.bronto.com").collect().map(_.getAs[Double](Score))
@@ -152,6 +174,8 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
val SuspiciousConnectsAnalysisResults(scoredData, _) =
ProxySuspiciousConnectsAnalysis.run(onlineTestConfigProxy, sparkSession, logger, data)
+ .getOrElse(SuspiciousConnectsAnalysisResults(sparkSession.sqlContext.emptyDataFrame,
+ sparkSession.sqlContext.emptyDataFrame))
val anomalyScore = scoredData.filter(scoredData(Host) === "intel.com").first().getAs[Double](Score)
val typicalScores = scoredData.filter(scoredData(Host) === "maw.bronto.com").collect().map(_.getAs[Double](Score))
@@ -202,6 +226,32 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
Math.abs(typicalScores(8) - 0.9d) should be <= 0.01d
}
+ it should "return None when input data frame doesn't matches expected schema" in {
+ val logger = LogManager.getLogger("SuspiciousConnectsAnalysis")
+ logger.setLevel(Level.WARN)
+
+
+ val anomalousRecord = ProxyInvalidInput("2016-10-03", "04:57:36", "127.0.0.1", 0.0d, "PUT",
+ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36",
+ 1, 230, "-", "Technology/Internet", "http://www.spoonflower.com/tags/color", "202", 80,
+ "/sites/c37i4q22szvir8ga3m8mtxaft7gwnm5fio8hfxo35mu81absi1/carts/4b3a313d-50f6-4117-8ffd-4e804fd354ef/fiddle",
+ "-", "127.0.0.1", 338, 647,
+ 1.0d)
+
+ val data = sparkSession.createDataFrame(Seq(anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord,
+ anomalousRecord))
+
+ val results =
+ ProxySuspiciousConnectsAnalysis.run(emTestConfigProxy, sparkSession, logger, data)
+
+ val none = results match {
+ case None => true
+ case Some(_) => false
+ }
+
+ none shouldBe true
+ }
+
"filterRecords" should "return data without garbage" in {
val cleanedProxyRecords = ProxySuspiciousConnectsAnalysis
@@ -232,6 +282,39 @@ class ProxySuspiciousConnectsAnalysisTest extends TestingSparkContextFlatSpec wi
}
+ "validateSchema" should "return false when passing an invalid schema" in {
+ val anomalousRecord = ProxyInvalidInput("2016-10-03", "04:57:36", "127.0.0.1", 0.0d, "PUT",
+ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36",
+ 1, 230, "-", "Technology/Internet", "http://www.spoonflower.com/tags/color", "202", 80,
+ "/sites/c37i4q22szvir8ga3m8mtxaft7gwnm5fio8hfxo35mu81absi1/carts/4b3a313d-50f6-4117-8ffd-4e804fd354ef/fiddle",
+ "-", "127.0.0.1", 338, 647,
+ 1.0d)
+
+ val data = sparkSession.createDataFrame(Seq(anomalousRecord, anomalousRecord, anomalousRecord, anomalousRecord,
+ anomalousRecord))
+
+ val results = ProxySuspiciousConnectsAnalysis.validateSchema(data)
+
+ results.isValid shouldBe false
+
+ }
+
+ it should "return true when passing a valid schema" in {
+ val typicalRecord = ProxyInput("2016-10-03", "04:57:36", "127.0.0.1", "maw.bronto.com", "PUT",
+ "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36",
+ "text/plain", 230, "-", "Technology/Internet", "http://www.spoonflower.com/tags/color", "202", 80,
+ "/sites/c37i4q22szvir8ga3m8mtxaft7gwnm5fio8hfxo35mu81absi1/carts/4b3a313d-50f6-4117-8ffd-4e804fd354ef/fiddle",
+ "-", "127.0.0.1", 338, 647,
+ "maw.bronto.com/sites/c37i4q22szvir8ga3m8mtxaft7gwnm5fio8hfxo35mu81absi1/carts/4b3a313d-50f6-4117-8ffd-4e804fd354ef/fiddle")
+
+ val data = sparkSession.createDataFrame(Seq(typicalRecord, typicalRecord, typicalRecord))
+
+ val results = ProxySuspiciousConnectsAnalysis.validateSchema(data)
+
+ results.isValid shouldBe true
+
+ }
+
def anomalousAndTypicalRecords(): (ProxyInput, ProxyInput) = {
val anomalousRecord = ProxyInput("2016-10-03", "04:57:36", "127.0.0.1", "intel.com", "PUT",
"Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36",
[04/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/ingest/ingest-summary.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/ingest/ingest-summary.html b/spot-oa/ui/ingest/ingest-summary.html
new file mode 100644
index 0000000..41324e1
--- /dev/null
+++ b/spot-oa/ui/ingest/ingest-summary.html
@@ -0,0 +1,132 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>Ingest Summary</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+ <style type="text/css">
+ .spot-row {
+ height: 100%;
+ }
+
+ .is-chart svg {
+ width: 100%;
+ height: 100%;
+ }
+
+ .is-chart svg .header text {
+ text-anchor: middle;
+ fill: #82837e;
+ }
+
+ .is-chart svg .header text tspan.bold {
+ font-weight: bold;
+ }
+
+ .is-chart .axis {
+ shape-rendering: crispEdges;
+ }
+
+ .is-chart .axis path, .is-chart .axis line {
+ fill: none;
+ }
+
+ .is-chart .pipeline {
+ pointer-events: all;
+ }
+
+ .is-chart .pipeline.zoom-in {
+ cursor: zoom-in;
+ }
+
+ .is-chart .pipeline.zoom-out {
+ cursor: zoom-out;
+ }
+
+ .is-chart .pipeline.e-resize {
+ cursor: e-resize;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: Ingest Summary</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../js/ingest-summary.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/ipython/custom/ajax-loader.gif
----------------------------------------------------------------------
diff --git a/spot-oa/ui/ipython/custom/ajax-loader.gif b/spot-oa/ui/ipython/custom/ajax-loader.gif
deleted file mode 100644
index 7dc5c7a..0000000
Binary files a/spot-oa/ui/ipython/custom/ajax-loader.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/ipython/custom/custom.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/ipython/custom/custom.css b/spot-oa/ui/ipython/custom/custom.css
deleted file mode 100755
index 0ae3a96..0000000
--- a/spot-oa/ui/ipython/custom/custom.css
+++ /dev/null
@@ -1,145 +0,0 @@
-.spot.spot_easy_mode {
- background-color: white;
-}
-
-.spot.spot_easy_mode:not(.spot_easy_mode_loading) #site {
- height: 100% !important;
-}
-
-.spot.spot_easy_mode:not(.spot_easy_mode_loading) #notebook {
- padding: 0 !important;
- box-shadow: none;
- -webkit-box-shadow: none;
-}
-
-.spot.spot_easy_mode #header,
-.spot.spot_easy_mode #pager,
-.spot.spot_easy_mode #tooltip,
-.spot.spot_easy_mode #notebook > .end_space,
-.spot.spot_easy_mode .cell:not(.code_cell),
-.spot.spot_easy_mode .cell.code_cell > .input,
-.spot.spot_easy_mode .cell.code_cell > .widget-area > .prompt {
- position: absolute !important;
- top: -1000000000px;
- left: -1000000000px;
-}
-
-.spot.spot_easy_mode.spot_easy_mode_loading .cell.code_cell {
- position: absolute !important;
- top: -1000000000px;
- left: -1000000000px;
-}
-
-.spot.spot_easy_mode #notebook-container {
- -webkit-box-shadow: none;
- box-shadow: none;
-}
-
-.spot.spot_easy_mode .cell.code_cell {
- border-color: transparent !important;
- padding: 0 !important;
-}
-
-/*****************************************
- ** EASY MODE SPINNER **
- *****************************************/
-.spot #spot_easy_mode_loader {
- display: none;
-}
-
-.spot.spot_easy_mode #spot_easy_mode_loader {
- display: block;
- position: fixed;
- top: 10px;
- width: 100%;
- text-align: center;
- margin-top: 50px;
-}
-
-.spot.spot_easy_mode #spot_easy_mode_loader_spinner
-{
- display: inline-block;
- width: 16px;
- height: 11px;
- margin-left: 5px;
- background-image: url(./ajax-loader.gif);
-}
-
-.spot.spot_easy_mode:not(.spot_easy_mode_loading) #spot_easy_mode_loader
-{
- right: 25px;
- margin-top: 0px;
- width: auto;
-}
-
-.spot.spot_ninja_mode #spot_easy_mode_loader
-{
- display: none;
-}
-
-/*****************************************
- ** Spot TOOLTIP STYLING **
- *****************************************/
-.spot .spot-tooltip .tooltip-inner {
- word-wrap: break-word;
-}
-
-.spot .spot-text-wrapper.overflown {
- color: red;
-}
-
-.spot .spot-text-wrapper {
- white-space: nowrap;
- overflow: hidden;
- text-overflow: ellipsis;
- width: 12em;
-}
-
-.spot .spot-text-wrapper.spot-text-sm {
- width: 6em;
-}
-
-.spot .spot-text-wrapper.spot-text-lg {
- width: 18em;
-}
-
-.spot .spot-text-wrapper.spot-text-xlg {
- width: 30em;
-}
-
-/* Classes for the threat investigation notebooks*/
-body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table,
-body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table{
- margin:10px 10px 20px 0px;
- font-size:12px;
-}
-
-
-body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td, th,
-body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td, th{
- border: solid 1px #CCC;
- padding: 3px;
- background-color: #FFF;
- line-height:14px;
-}
-
-body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td,
-body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td{
- height:25px;
- max-width: 180px;
- width:100px;
-}
-
-body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table th,
-body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table th{
-
- background-color: #0071C5;
- color: #FFF;
- text-align: center;
-}
-
-body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area > .widget-subarea,
-body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area > .widget-subarea
-{
- width: 90%;
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/ipython/custom/custom.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/ipython/custom/custom.js b/spot-oa/ui/ipython/custom/custom.js
deleted file mode 100755
index 2496d59..0000000
--- a/spot-oa/ui/ipython/custom/custom.js
+++ /dev/null
@@ -1,337 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-/**
- * Trigger reload event on parent document
- */
-function reloadParentData()
-{
- window.parent.iframeReloadHook && window.parent.iframeReloadHook();
-}
-
-require(['jquery', 'bootstrap'], function($, bootstrap)
-{
- function retry(query) {
- var regex = new RegExp(query + '=(\\d)');
-
- var matcher = regex.exec(window.location.hash);
-
- var attempt = matcher ? ++matcher[1] : 2;
-
- // Set hash
- window.location.hash = '#' + query + '=' + attempt;
- window.location.reload();
- }
-
- // jQuery must be available for easy mode to woark properly
- if ($===undefined) {
- if (/#spot_jquery_retry=[015-9]/.test(window.location.hash)) {
- alert('Notebook was not able to load jQuery after 5 attempts. Please try again later, this is a known issue.');
- console.warn('Bootstrap\'s tooltip was not loaded.');
- }
- else {
- confirm('Notebook failed to load jQuery, easy mode would not work properly without it. Would you you like to try again?')
- && retry('spot_jquery_retry');
- }
-
- return;
- }
-
- if ($.fn.tooltip===undefined) {
- if (/#spot_bootstrap_retry=\d/.test(window.location.hash)) {
- alert('Notebook was not able to load bootstrap\'s tooltip plugin. You can still work withuot this feature.');
- } else {
- confirm('Notebook was not able to load bootstrap\'s tooltip plugin. You can still work withuot this feature. Would you like to try again?')
- && retry('spot_bootstrap_retry');
- }
- }
-
- var easyMode = {
- stage: 0,
- KERNEL_READY: 1,
- NOTEBOOK_READY: 2,
- DOM_READY: 4,
- ENV_READY: 7,
- building: false,
- present: false,
- ready: false,
- cells: {
- total: 0,
- execution_queue: []
- }
- }
-
- /**
- * Show EasyMode
- *
- * Hide everything but IPython form
- */
- function showEasyMode()
- {
- $(document.body).addClass('spot_easy_mode').removeClass('spot_ninja_mode');
- }
-
- /**
- * Hide Easy Mode
- *
- * Show full IPython Notebook
- */
- function hideEasyMode()
- {
- $(document.body).addClass('spot_ninja_mode').removeClass('spot_easy_mode');
- }
-
- function insertProgressIndicator()
- {
- $(document.body).append(
- '<div id="spot_easy_mode_loader">' +
- '<span id="spot_easy_mode_loader_details"></span>' +
- '<span id="spot_easy_mode_loader_spinner"></span>' +
- '</div>'
- );
- }
-
- /**
- * Remove progress indicator
- */
- function removeProgressIndicator()
- {
- $(document.body).removeClass('spot_easy_mode_loading');
-
- $('#spot_easy_mode_loader').remove();
- }
-
- /**
- * Updates progress indicator's details
- */
- function updateProgressIndicator(content)
- {
- $('#spot_easy_mode_loader_details').html(content);
- }
-
- /**
- * Show a progress indicator to users
- */
- function showBuildingUiIndicator()
- {
- $(document.body).addClass('spot_easy_mode_loading');
-
- insertProgressIndicator();
-
- updateProgressIndicator(
- 'Building UI <span id="spot_easy_mode_loader_progress">0</span>%'
- );
- }
-
- /**
- * Update progress indicator
- */
- function updateBuildingUiIndicator()
- {
- var p;
-
- p = (easyMode.cells.total-easyMode.cells.execution_queue.length) * 100 / easyMode.cells.total;
-
- $('#spot_easy_mode_loader_progress').text(Math.floor(p));
- }
-
- function easyModeBootStrap (IPython)
- {
- if (easyMode.stage!=easyMode.ENV_READY) return;
-
- // 1 Build widgets
- easyMode.building = true;
-
- console.info('Spot: Building easy mode...');
-
- // 2 Create an execution queue to display progress
- easyMode.cells.execution_queue = [];
-
- easyMode.cells.total = 0;
- IPython.notebook.get_cells().forEach(function (cell)
- {
- if (cell.cell_type==='code')
- {
- easyMode.cells.total++;
- }
- });
-
- // Add an extra cell to show progress when requesting execution
- easyMode.cells.total++;
-
- // 3 Execute all cells ( Generate UI )
- IPython.notebook.execute_all_cells();
-
- updateBuildingUiIndicator();
- }
-
- function isEasyModeAvailable()
- {
- return window.parent!=window;
- }
-
- function isEasyModeEnabled() {
- return /showEasyMode/.test(window.location.hash);
- }
-
- function isNinjaModeEnabled() {
- return /showNinjaMode/.test(window.location.hash);
- }
-
- if (isEasyModeAvailable()) {
- // Add spot CSS classes (easymode enabled by default)
- $(document.body).addClass('spot').addClass('spot_easy_mode').addClass('spot_easy_mode_loading');
-
- // Listen for URL's hash changes
- $(window).on('hashchange', function ()
- {
- if (isEasyModeEnabled()) showEasyMode();
- else if (isNinjaModeEnabled()) hideEasyMode();
- });
-
- $(function () {
- // Show progress indicator
- showBuildingUiIndicator();
- });
- }
-
- // Enable spot tooltip text wrapper
- $(function () {
- $('body').tooltip({
- selector: '.spot-text-wrapper[data-toggle]',
- container: 'body',
- template: '<div class="spot-tooltip tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',
- title: function () {
- return $(this).html();
- },
- html: true
- });
-
- $('body').on('show.bs.tooltip', '.spot-text-wrapper', function () {
- return this.clientWidth !== this.scrollWidth || this.clientHeight !== this.scrollHeight;
- });
- });
-
- /**
- * The following code enables toggle from normal user mode (wizard) and ninja node (notebook UI)
- */
- require(['base/js/namespace', 'base/js/events'], function(IPython, events)
- {
- // Do nothing when running stand alone
- if (!isEasyModeAvailable()) return;
-
- // We are running inside and iframe from Spot. Let's have some fun!
-
- // Let Notebook be aware it is running on an iframe
- IPython._target = '_self';
-
- events.on('kernel_busy.Kernel', function ()
- {
- // Skip this event while building UI
- if (!easyMode.ready) return;
-
- $('#notebook button.btn:not([disabled])').addClass('spotDisabled').attr('disabled', 'disabled');
-
- insertProgressIndicator();
- });
-
- events.on('kernel_idle.Kernel', function ()
- {
- // Skip this event while building UI
- if (!easyMode.ready) return;
-
- removeProgressIndicator();
-
- $('#notebook button.btn.spotDisabled').removeClass('spotDisabled').removeAttr('disabled');
- });
-
- events.on('kernel_ready.Kernel', function ()
- {
- console.info('Spot: Kernel is ready');
-
- easyMode.stage |= easyMode.KERNEL_READY;
-
- easyModeBootStrap(IPython);
- });
-
- events.on('notebook_loaded.Notebook', function ()
- {
- console.info('Spot: Notebook loaded');
-
- easyMode.stage |= easyMode.NOTEBOOK_READY;
-
- easyModeBootStrap(IPython);
- });
-
- events.on('shell_reply.Kernel', function (evt, data)
- {
- var reply, cell, cellIdx;
-
- reply = data.reply;
- cell = easyMode.cells.execution_queue.shift();
-
- console.log('Last execution status: ' + reply.content.status);
-
- if ((easyMode.building || easyMode.ready) && reply.content.status==='error')
- {
- // First error found
- easyMode.building = false;
- easyMode.ready = false;
- isEasyModeEnabled() && alert('Ooops some code failed. Please go to ipython notebook mode and manually fix the error.');
- $(document.body).removeClass('spot');
- removeProgressIndicator();
- hideEasyMode();
- // Select and scroll to first cell with errors
- cellIdx = IPython.notebook.find_cell_index(cell);
- IPython.notebook.scroll_to_cell(cellIdx);
- IPython.notebook.select(cellIdx);
- }
-
- if (!easyMode.building)
- {
- return;
- }
-
- if (easyMode.cells.execution_queue.length===0)
- {
- console.info('Spot: Cell execution has finished');
-
- easyMode.ready = true;
- easyMode.building = false;
-
- removeProgressIndicator();
- isEasyModeEnabled() && showEasyMode();
- }
- else
- {
- updateBuildingUiIndicator();
- }
- });
-
- events.on('execute.CodeCell', function (ev, obj)
- {
- var cell;
-
- if (!easyMode.building && !easyMode.ready) return;
-
- cell = obj.cell;
-
- easyMode.cells.execution_queue.push(cell);
-
- console.info('Spot: Cell execution requested: ' + easyMode.cells.execution_queue.length + ' of ' + easyMode.cells.total);
-
- cell.clear_output(false);
- // There seems to be a bug on IPython sometimes cells with widgets dont get cleared
- // Workaround:
- cell.output_area.clear_output(false, true);
- });
-
- $(function ()
- {
- console.info('Spot: DOM is ready');
-
- easyMode.stage |= easyMode.DOM_READY;
-
- easyModeBootStrap(IPython);
- });
- });
-});
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/actions/EdInActions.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/actions/EdInActions.js b/spot-oa/ui/js/actions/EdInActions.js
index 4e9d4e8..4fb39d8 100755
--- a/spot-oa/ui/js/actions/EdInActions.js
+++ b/spot-oa/ui/js/actions/EdInActions.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var SpotDispatcher = require('../dispatchers/SpotDispatcher');
var SpotActions = require('./SpotActions');
@@ -58,6 +73,23 @@ var EdInActions = {
ip: ip
});
},
+ saveScoring: function(scoredEmelents) {
+ SpotDispatcher.dispatch({
+ actionType: SpotConstants.SAVE_SCORED_ELEMENTS,
+ scoredElems: scoredEmelents
+ });
+ },
+ resetScoring: function(date) {
+ SpotDispatcher.dispatch({
+ actionType: SpotConstants.RESET_SCORED_ELEMENTS,
+ date: date
+ });
+ },
+ setClassWidth: function() {
+ SpotDispatcher.dispatch({
+ actionType: SpotConstants.CHANGE_CSS_CLS
+ });
+ }
};
module.exports = EdInActions;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/actions/InSumActions.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/actions/InSumActions.js b/spot-oa/ui/js/actions/InSumActions.js
index ea5cbe0..8c439ac 100755
--- a/spot-oa/ui/js/actions/InSumActions.js
+++ b/spot-oa/ui/js/actions/InSumActions.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotDispatcher = require('../dispatchers/SpotDispatcher');
const SpotConstants = require('../constants/SpotConstants');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/actions/SpotActions.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/actions/SpotActions.js b/spot-oa/ui/js/actions/SpotActions.js
index a70f2a9..6cb3f0d 100755
--- a/spot-oa/ui/js/actions/SpotActions.js
+++ b/spot-oa/ui/js/actions/SpotActions.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var $ = require('jquery');
@@ -35,6 +50,12 @@ var SpotActions = {
name: name
});
},
+ setPipeline(pipeline) {
+ SpotDispatcher.dispatch({
+ actionType: SpotConstants.UPDATE_PIPELINE,
+ pipeline
+ });
+ },
expandPanel: function (panel) {
SpotDispatcher.dispatch({
actionType: SpotConstants.EXPAND_PANEL,
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/actions/StoryboardActions.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/actions/StoryboardActions.js b/spot-oa/ui/js/actions/StoryboardActions.js
index bf60b7c..13fb5e6 100755
--- a/spot-oa/ui/js/actions/StoryboardActions.js
+++ b/spot-oa/ui/js/actions/StoryboardActions.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var SpotDispatcher = require('../dispatchers/SpotDispatcher');
var SpotConstants = require('../constants/SpotConstants');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/actions/__tests__/EdInActions.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/actions/__tests__/EdInActions.test.js b/spot-oa/ui/js/actions/__tests__/EdInActions.test.js
index 0732997..ac13819 100644
--- a/spot-oa/ui/js/actions/__tests__/EdInActions.test.js
+++ b/spot-oa/ui/js/actions/__tests__/EdInActions.test.js
@@ -1,4 +1,17 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
jest.mock('../../dispatchers/SpotDispatcher');
jest.mock('../SpotActions');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/actions/__tests__/SpotActions.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/actions/__tests__/SpotActions.test.js b/spot-oa/ui/js/actions/__tests__/SpotActions.test.js
index ebaf30c..aabc119 100644
--- a/spot-oa/ui/js/actions/__tests__/SpotActions.test.js
+++ b/spot-oa/ui/js/actions/__tests__/SpotActions.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
jest.mock('../../dispatchers/SpotDispatcher');
jest.mock('../../utils/SpotUtils');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/actions/__tests__/StoryboardActions.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/actions/__tests__/StoryboardActions.test.js b/spot-oa/ui/js/actions/__tests__/StoryboardActions.test.js
index 377e6fd..1d1c390 100644
--- a/spot-oa/ui/js/actions/__tests__/StoryboardActions.test.js
+++ b/spot-oa/ui/js/actions/__tests__/StoryboardActions.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
jest.mock('../../dispatchers/SpotDispatcher');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/ChartMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/ChartMixin.react.js b/spot-oa/ui/js/components/ChartMixin.react.js
index 210c3c9..edcf205 100755
--- a/spot-oa/ui/js/components/ChartMixin.react.js
+++ b/spot-oa/ui/js/components/ChartMixin.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const $ = require('jquery');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/ContentLoaderMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/ContentLoaderMixin.react.js b/spot-oa/ui/js/components/ContentLoaderMixin.react.js
index ba1b98e..aa34b71 100755
--- a/spot-oa/ui/js/components/ContentLoaderMixin.react.js
+++ b/spot-oa/ui/js/components/ContentLoaderMixin.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/DateInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/DateInput.react.js b/spot-oa/ui/js/components/DateInput.react.js
index 72f89b4..e20d646 100755
--- a/spot-oa/ui/js/components/DateInput.react.js
+++ b/spot-oa/ui/js/components/DateInput.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var $ = require('jquery');
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/DendrogramMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/DendrogramMixin.react.js b/spot-oa/ui/js/components/DendrogramMixin.react.js
index 74328f4..9eca114 100755
--- a/spot-oa/ui/js/components/DendrogramMixin.react.js
+++ b/spot-oa/ui/js/components/DendrogramMixin.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var $ = require('jquery');
const Base64 = require('js-base64').Base64;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/DetailsGridMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/DetailsGridMixin.react.js b/spot-oa/ui/js/components/DetailsGridMixin.react.js
index 1213581..6598552 100755
--- a/spot-oa/ui/js/components/DetailsGridMixin.react.js
+++ b/spot-oa/ui/js/components/DetailsGridMixin.react.js
@@ -1,5 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var DetailsGridMixin = {
emptySetMessage: 'Please select one row from Suspicious Frame',
getInitialState: function ()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/ExecutiveThreatBriefingPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/ExecutiveThreatBriefingPanel.react.js b/spot-oa/ui/js/components/ExecutiveThreatBriefingPanel.react.js
index e1964c5..8d66957 100755
--- a/spot-oa/ui/js/components/ExecutiveThreatBriefingPanel.react.js
+++ b/spot-oa/ui/js/components/ExecutiveThreatBriefingPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/GridPanelMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/GridPanelMixin.react.js b/spot-oa/ui/js/components/GridPanelMixin.react.js
index c116dd4..99c549c 100755
--- a/spot-oa/ui/js/components/GridPanelMixin.react.js
+++ b/spot-oa/ui/js/components/GridPanelMixin.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/IPythonNotebookPanel.react.js b/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
index 424d6ce..f5af7ce 100755
--- a/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
+++ b/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
@@ -1,10 +1,26 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
var SpotActions = require('../actions/SpotActions');
var SpotConstants = require('../constants/SpotConstants');
var SpotStore = require('../stores/SpotStore');
+var easyModeS = true;
var IPythonNotebookPanel = React.createClass({
propTypes: {
@@ -14,10 +30,11 @@ var IPythonNotebookPanel = React.createClass({
},
statics: {
createIPythonNotebookClosure: function (title, easyMode) {
- var closure;
-
+ var closure;
+
easyMode = typeof easyMode=='undefined' ? true : typeof easyMode=='boolean' ? easyMode : !!easyMode;
+ easyModeS = easyMode;
closure = {
getTitle: function () {
return title;
@@ -57,7 +74,7 @@ var IPythonNotebookPanel = React.createClass({
}
},
getInitialState: function () {
- return {date: this.props.date.replace(/-/g, ''), easyMode: true};
+ return {date: this.props.date.replace(/-/g, ''), easyMode: easyModeS};
},
componentDidMount: function () {
SpotStore.addChangeDateListener(this._onDateChange);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/IngestSummaryPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/IngestSummaryPanel.react.js b/spot-oa/ui/js/components/IngestSummaryPanel.react.js
index 6e2b98d..bc8c48f 100755
--- a/spot-oa/ui/js/components/IngestSummaryPanel.react.js
+++ b/spot-oa/ui/js/components/IngestSummaryPanel.react.js
@@ -1,14 +1,27 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const $ = require('jquery');
const d3 = require('d3');
const React = require('react');
-const ReactDOM = require('react-dom');
const ContentLoaderMixin = require('./ContentLoaderMixin.react');
const ChartMixin = require('./ChartMixin.react');
const DateUtils = require('../utils/DateUtils');
-const InSumActions = require('../actions/InSumActions');
const IngestSummaryStore = require('../stores/IngestSummaryStore');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/OptionPicker.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/OptionPicker.react.js b/spot-oa/ui/js/components/OptionPicker.react.js
index 44f4166..5c4ce9f 100644
--- a/spot-oa/ui/js/components/OptionPicker.react.js
+++ b/spot-oa/ui/js/components/OptionPicker.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/Panel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/Panel.react.js b/spot-oa/ui/js/components/Panel.react.js
index 732b62b..a746edc 100755
--- a/spot-oa/ui/js/components/Panel.react.js
+++ b/spot-oa/ui/js/components/Panel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
var assign = require('object-assign');
@@ -39,6 +54,17 @@ var Panel = React.createClass({
if (this.props.header) {
buttons = [];
+ if (this.props.switchable) {
+ buttons.push(
+ <li key="switchBtn">
+ <button type="button" className="btn btn-default btn-xs pull-right"
+ onClick={this.props.onSwitch}>
+ <span className="glyphicon glyphicon-sort" aria-hidden="true"></span>
+ </button>
+ </li>
+ );
+ }
+
if (this.props.reloadable) {
buttons.push(
<li key="realoadBtn" className="refresh">
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/PanelRow.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/PanelRow.react.js b/spot-oa/ui/js/components/PanelRow.react.js
index ca10d38..4d8bdb8 100755
--- a/spot-oa/ui/js/components/PanelRow.react.js
+++ b/spot-oa/ui/js/components/PanelRow.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
@@ -32,9 +47,10 @@ var PanelRow = React.createClass({
},
render: function () {
var cssCls = this.state.maximized ? 'spot-maximized' : this.state.minimized ? 'spot-minimized' : '';
+ var sortable = {'order':1};
return (
- <div className={'spot-row row ' + cssCls}>
+ <div id={this.props.title || ''} className={'spot-row row fit ' + cssCls} style={sortable}>
{this.props.children}
</div>
);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/PolloNetworkViewMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/PolloNetworkViewMixin.react.js b/spot-oa/ui/js/components/PolloNetworkViewMixin.react.js
index 09b684c..e31cfa9 100755
--- a/spot-oa/ui/js/components/PolloNetworkViewMixin.react.js
+++ b/spot-oa/ui/js/components/PolloNetworkViewMixin.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const Base64 = require('js-base64').Base64;
const d3Interpolate = require('d3-interpolate');
@@ -19,7 +34,7 @@ const PolloNetworkViewMixin = {
// Node related scales
const hitsDomain = [1, this.state.data.maxNodes];
- const ipDomain = [false, true];
+ const ipDomain = [0,1];
this.nodeSizeScale = d3.scale.linear().domain(hitsDomain);
this.chargeScale = d3.scale.linear().domain(hitsDomain);
this.typeScale = d3.scale.ordinal().domain(ipDomain).range(["circle", "diamond"]);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/SuspiciousGridMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/SuspiciousGridMixin.react.js b/spot-oa/ui/js/components/SuspiciousGridMixin.react.js
index e42ad8d..297978f 100755
--- a/spot-oa/ui/js/components/SuspiciousGridMixin.react.js
+++ b/spot-oa/ui/js/components/SuspiciousGridMixin.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var $ = require('jquery');
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/TimelineMixin.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/TimelineMixin.react.js b/spot-oa/ui/js/components/TimelineMixin.react.js
index 8d28830..3c568a5 100755
--- a/spot-oa/ui/js/components/TimelineMixin.react.js
+++ b/spot-oa/ui/js/components/TimelineMixin.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const $ = require('jquery');
const d3 = require('d3');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/scoring/ButtonsScoring.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/scoring/ButtonsScoring.react.js b/spot-oa/ui/js/components/scoring/ButtonsScoring.react.js
new file mode 100644
index 0000000..39ebcff
--- /dev/null
+++ b/spot-oa/ui/js/components/scoring/ButtonsScoring.react.js
@@ -0,0 +1,48 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var ButtonsScoring = React.createClass({
+ getInitialState: function() {
+ return {dataScored: []};
+ },
+ render: function() {
+
+ let classNameCol, classNameBtn;
+
+ if(this.props.action === 'reset') {
+ classNameCol = `col-md-${this.props.col} col-xs-12 col-lg-${this.props.col} col-md-offset-3 col-lg-offset-3`;
+ classNameBtn = "btn col-md-12 col-xs-12 col-lg-12";
+ } else {
+ classNameCol = `col-md-${this.props.col} col-xs-6 col-lg-${this.props.col}`;
+ classNameBtn = "btn btn-primary col-md-12 col-xs-12 col-lg-12";
+ }
+
+ return(
+ <div className={classNameCol}>
+ <button className={classNameBtn} onClick={this.checkAction}>{this.props.name}</button>
+ </div>
+ );
+
+ },
+ checkAction: function() {
+ this.props.onChange();
+ }
+});
+
+module.exports = ButtonsScoring;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/scoring/FilterSelectInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/scoring/FilterSelectInput.react.js b/spot-oa/ui/js/components/scoring/FilterSelectInput.react.js
new file mode 100644
index 0000000..9b76377
--- /dev/null
+++ b/spot-oa/ui/js/components/scoring/FilterSelectInput.react.js
@@ -0,0 +1,45 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+var SpotUtils = require('../../utils/SpotUtils');
+
+var FilterSelectInput = React.createClass({
+ getInitialState: function() {
+ return {filterBox: ''}
+ },
+ render: function() {
+
+ return(
+ <div className={`col-md-${this.props.col} col-xs-12 col-sm-6 col-lg-${this.props.col}`}>
+ <div className="inner-addon right-addon">
+ <i className="glyphicon glyphicon-search"></i>
+ <input className="form-control filter-select" type="text" maxLength="15" placeholder={this.props.nameBox} id={this.props.idInput} autoFocus={false} onChange={this._onChange} value={this.state.filterBox} />
+ </div>
+ </div>
+ )
+ },
+ _onChange: function (e)
+ {
+ this.setState({filterBox: e.target.value});
+ SpotUtils.filterTextOnSelect($(this.props.idSelect), e.target.value);
+ },
+
+});
+
+
+module.exports = FilterSelectInput;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/scoring/RatingInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/scoring/RatingInput.react.js b/spot-oa/ui/js/components/scoring/RatingInput.react.js
new file mode 100644
index 0000000..9da6db2
--- /dev/null
+++ b/spot-oa/ui/js/components/scoring/RatingInput.react.js
@@ -0,0 +1,42 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var RatingInput = React.createClass({
+ render: function() {
+ var radioOptions = this.props.data.map((obj, i) =>
+ <label className="radio-inline">
+ <input key={i} type="radio" name={obj.radioName} id="rating" value={obj.value} defaultChecked={obj.selected}/>{obj.name}
+ </label>
+ );
+
+ return(
+ <div className={`col-md-${this.props.col} col-lg-${this.props.col} col-xs-12`}>
+ <div className="col-md-2 col-lg-2 col-xs-12">
+ <label>Rating: </label>
+ </div>
+ <div className="col-md-10 col-lg-10 col-xs-12">
+ {radioOptions}
+ </div>
+ </div>
+ );
+ }
+});
+
+
+module.exports = RatingInput;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/scoring/ScoreMessage.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/scoring/ScoreMessage.react.js b/spot-oa/ui/js/components/scoring/ScoreMessage.react.js
new file mode 100644
index 0000000..37beebe
--- /dev/null
+++ b/spot-oa/ui/js/components/scoring/ScoreMessage.react.js
@@ -0,0 +1,38 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var ScoreMessage = React.createClass({
+ getInitialState : function() {
+ return { showMe : false };
+ },
+
+ render: function() {
+ return(
+ <div className="text-center hidden" id={this.props.who}>
+ <label className="text-danger">"Click the 'Save' button when you’re finished scoring"</label>
+ </div>
+ );
+ },
+
+ checkAction: function(value) {
+ this.setState({ showMe : value}) ;
+ }
+});
+
+module.exports = ScoreMessage;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/scoring/SearchGlobalInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/scoring/SearchGlobalInput.react.js b/spot-oa/ui/js/components/scoring/SearchGlobalInput.react.js
new file mode 100644
index 0000000..664fcfa
--- /dev/null
+++ b/spot-oa/ui/js/components/scoring/SearchGlobalInput.react.js
@@ -0,0 +1,49 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var SearchGlobalInput = React.createClass({
+ getInitialState: function() {
+ return {filter: ''}
+ },
+ render: function() {
+ return(
+ <div className={`col-md-${this.props.col} col-lg-${this.props.col} col-xs-12`}>
+ <div className="inner-addon right-addon">
+ <i className="glyphicon glyphicon-search"></i>
+ <input className="form-control" type="text" placeholder="Quick scoring..." maxLength={this.props.maxlength || 15} id="globalTxt" autoFocus={false} onChange={this._onChange} value={this.state.filter} onKeyUp={this._onKeyUp} />
+ </div>
+ </div>
+ );
+ },
+ // _onKeyUp: function (e)
+ // {
+ // this.setState({filterBox: e.target.value})
+ // },
+ _onChange: function (e)
+ {
+ this.setState({filter: e.target.value})
+ },
+ _onFilterChange: function ()
+ {
+ console.log(e.target)
+ }
+});
+
+
+module.exports = SearchGlobalInput;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/components/scoring/SelectInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/scoring/SelectInput.react.js b/spot-oa/ui/js/components/scoring/SelectInput.react.js
new file mode 100644
index 0000000..321b154
--- /dev/null
+++ b/spot-oa/ui/js/components/scoring/SelectInput.react.js
@@ -0,0 +1,46 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var SelectInput = React.createClass({
+ propTypes: {
+ title: React.PropTypes.string.isRequired
+ },
+ componentDidMount: function() {
+ // $('.panel-body-container').height();
+ },
+ render: function() {
+
+ var labelOptions = this.props.options.map((data, i) =>
+ <option key={i} value={data}>{data}</option>
+ );
+
+
+ return(
+ <div className={`text-left col-md-${this.props.col} col-xs-12 col-sm-6 col-lg-${this.props.col}`}>
+ <select size="8" className="col-md-12 select-picker form-control" id={this.props.who} onChange={this.logChange}>
+ <option value="" selected>- Select -</option>
+ {labelOptions}
+ </select>
+ </div>
+ );
+ }
+});
+
+
+module.exports = SelectInput;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/constants/SpotConstants.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/constants/SpotConstants.js b/spot-oa/ui/js/constants/SpotConstants.js
index 08f3335..448068e 100755
--- a/spot-oa/ui/js/constants/SpotConstants.js
+++ b/spot-oa/ui/js/constants/SpotConstants.js
@@ -1,9 +1,25 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotConstants = {
PIPELINE_NETFLOW: 'flow',
PIPELINE_DNS: 'dns',
PIPELINE_PROXY: 'proxy',
+ UPDATE_PIPELINE: 'UPDATE_PIPELINE',
// Search Actions
UPDATE_FILTER: 'UPDATE_FILTER',
UPDATE_DATE: 'UPDATE_DATE',
@@ -17,6 +33,11 @@ const SpotConstants = {
SUSPICIOUS_PANEL:'Suspicious',
NETVIEW_PANEL: 'Network View',
NOTEBOOK_PANEL: 'Notebook',
+ NO_IPYTHON_NOTEBOOK: 'NoIpythonNotebooks',
+ SCORING_PANEL: 'Scoring',
+ SAVE_SCORED_ELEMENTS: 'SAVE_SCORED_ELEMENTS',
+ RESET_SCORED_ELEMENTS: 'RESET_SCORED_ELEMENTS',
+ CHANGE_CSS_CLS: 'CHANGE_CSS_CLS',
DETAILS_PANEL: 'Details',
COMMENTS_PANEL:'Executive Threat Briefing',
INCIDENT_PANEL:'Incident Progression',
@@ -42,7 +63,8 @@ const SpotConstants = {
START_DATE: 'start-date',
END_DATE: 'end-date',
// Server Paths
- NOTEBOOKS_PATH: '/notebooks/ipynb'
+ NOTEBOOKS_PATH: '/notebooks/ipynb',
+ GRAPHQL_ENDPOINT: '/graphql'
};
module.exports = SpotConstants;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/cubehelix/cubehelix.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/cubehelix/cubehelix.js b/spot-oa/ui/js/cubehelix/cubehelix.js
index 4a68fa0..cf13cc0 100755
--- a/spot-oa/ui/js/cubehelix/cubehelix.js
+++ b/spot-oa/ui/js/cubehelix/cubehelix.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
(function() {
var radians = Math.PI / 180;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/dispatchers/SpotDispatcher.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/dispatchers/SpotDispatcher.js b/spot-oa/ui/js/dispatchers/SpotDispatcher.js
index ea57c1c..e001161 100755
--- a/spot-oa/ui/js/dispatchers/SpotDispatcher.js
+++ b/spot-oa/ui/js/dispatchers/SpotDispatcher.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var Dispatcher = require('flux').Dispatcher;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/ingest-summary.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/ingest-summary.js b/spot-oa/ui/js/ingest-summary.js
index f907477..7afecd8 100755
--- a/spot-oa/ui/js/ingest-summary.js
+++ b/spot-oa/ui/js/ingest-summary.js
@@ -1,10 +1,26 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
const SpotActions = require('./actions/SpotActions');
const InSumActions = require('./actions/InSumActions');
+const EdInActions = require('./actions/EdInActions');
const IngestSummaryStore = require('./stores/IngestSummaryStore');
const SpotConstants = require('./constants/SpotConstants');
const SpotUtils = require('./utils/SpotUtils');
@@ -13,6 +29,7 @@ const DateUtils = require('./utils/DateUtils');
// Build and Render Toolbar
const DateInput = require('./components/DateInput.react');
const OptionPicker = require('./components/OptionPicker.react');
+const MainMenu = require('./menu/components/MainMenu.react');
// Find out period
var startDate, endDate, today;
@@ -44,16 +61,23 @@ if (endDate < startDate)
startDate = endDate;
endDate = today;
}
-
const PIPELINES = IngestSummaryStore.PIPELINES;
-const DEFAULT_PIPELINE = Object.keys(PIPELINES)[0];
+//check if pipeline is on URL, if not the first element of PIPELINES is taken
+const DEFAULT_PIPELINE = SpotUtils.getUrlParam('pipeline') || Object.keys(PIPELINES)[0];
const loadPipeline = function loadPipeline(pipeline) {
- IngestSummaryStore.setPipeline(pipeline);
+ SpotActions.setPipeline(pipeline);
InSumActions.reloadSummary();
}
ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+SpotActions.setDate(SpotUtils.getCurrentDate());
+
+ReactDOM.render(
<form className="form-inline">
<div className="form-group">
<label htmlFor="pipeline-picker">Source: </label>
@@ -106,9 +130,12 @@ ReactDOM.render(
document.getElementById('spot-content-wrapper')
);
+
+
// Set period
SpotActions.setDate(startDate, SpotConstants.START_DATE);
SpotActions.setDate(endDate, SpotConstants.END_DATE);
+SpotActions.setPipeline(DEFAULT_PIPELINE);
// Load data
loadPipeline(DEFAULT_PIPELINE);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/menu/components/MainMenu.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/menu/components/MainMenu.react.js b/spot-oa/ui/js/menu/components/MainMenu.react.js
new file mode 100644
index 0000000..78f9c72
--- /dev/null
+++ b/spot-oa/ui/js/menu/components/MainMenu.react.js
@@ -0,0 +1,73 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var MainMenuStore = require('../stores/MainMenuStore');
+
+var MainMenu = React.createClass({
+ getInitialState: function () {
+ return {
+ menu: MainMenuStore.getMenu()
+ };
+ },
+ componentDidMount: function () {
+ MainMenuStore.addChangeMenuListener(this.onChange);
+ },
+ componentWillUnmount: function () {
+ MainMenuStore.removeChangeMenuListener(this.onChange);
+ },
+ render: function () {
+ let createdMenu = this.state.menu.map((dropdwn) =>
+ <li className="dropdown">
+ {(dropdwn.sub.length === 0 && dropdwn.labelledby === 'pluginsMenu') ? '' : dropdwn.sub.length === 0 ?
+ <a data-href={dropdwn.link} target={dropdwn.target}>{dropdwn.name}</a>
+ :
+ <a className="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
+ {dropdwn.glyphicon !== '' ? <span className={'glyphicon ' + dropdwn.glyphicon} aria-hidden="true"></span>
+ :
+ dropdwn.name}
+ <span className="caret"></span>
+ </a>
+ }
+ <ul className="dropdown-menu" aria-labelledby={dropdwn.labelledby}>
+ {dropdwn.sub.map((li) =>
+ <li>
+ <a data-href={li.link} target={li.target}>{li.name}</a>
+ </li>
+ )}
+ </ul>
+ </li>
+ );
+
+ return(
+ <div>
+ <ul className="nav navbar-nav navbar-right">
+ {createdMenu}
+ </ul>
+ </div>
+ );
+ },
+ onChange: function ()
+ {
+ this.setState({menu: MainMenuStore.getMenu()});
+ }
+
+});
+
+
+module.exports = MainMenu;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/menu/menu.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/menu/menu.js b/spot-oa/ui/js/menu/menu.js
new file mode 100644
index 0000000..ac00647
--- /dev/null
+++ b/spot-oa/ui/js/menu/menu.js
@@ -0,0 +1,52 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+let menu =
+ {
+ menu:
+ [
+ {name: 'Flows', link: '', glyphicon: '', labelledby: 'flowsMenu', sub:
+ [
+ {name: 'Suspicious', link: '../flow/suspicious.html#date=${date}', target: '_self'},
+ {name: 'Threat Investigation', link: '../flow/threat-investigation.html#date=${date}', target: '_self'},
+ {name: 'Storyboard', link: '../flow/storyboard.html#date=${date}', target: '_self'},
+ {name: 'Advanced Mode', link: '../flow/ipython_notebook.html#date=${date}', target: '_blank'}
+ ]
+ },
+ {name: 'DNS', link: '', glyphicon: '', labelledby: 'dnsMenu', sub:
+ [
+ {name: 'Suspicious', link: '../dns/suspicious.html#date=${date}', target: '_self'},
+ {name: 'Threat Investigation', link: '../dns/threat-investigation.html#date=${date}', target: '_self'},
+ {name: 'Storyboard', link: '../dns/storyboard.html#date=${date}', target: '_self'},
+ {name: 'Advanced Mode', link: '../dns/ipython_notebook.html#date=${date}', target: '_blank'}
+ ]
+ },
+ {name: 'Proxy', link: '', glyphicon: '', labelledby: 'proxyMenu', sub:
+ [
+ {name: 'Suspicious', link: '../proxy/suspicious.html#date=${date}', target: '_self'},
+ {name: 'Threat Investigation', link: '../proxy/threat-investigation.html#date=${date}', target: '_self'},
+ {name: 'Storyboard', link: '../proxy/storyboard.html#date=${date}', target: '_self'},
+ {name: 'Advanced Mode', link: '../proxy/ipython_notebook.html#date=${date}', target: '_blank'}
+ ]
+ },
+ {name: 'Ingest Summary', link: '../ingest/ingest-summary.html#end-date=${date}', glyphicon: '', labelledby: '', sub: [], target: '_self'},
+ {name: 'Plugins', link: '', glyphicon: '', labelledby: 'pluginsMenu', sub: []}
+
+ ]
+ };
+
+module.exports = menu;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/menu/stores/MainMenuStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/menu/stores/MainMenuStore.js b/spot-oa/ui/js/menu/stores/MainMenuStore.js
new file mode 100644
index 0000000..2253500
--- /dev/null
+++ b/spot-oa/ui/js/menu/stores/MainMenuStore.js
@@ -0,0 +1,57 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+const EventEmitter = require('events').EventEmitter;
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const Menu = require('../menu');
+
+class MainMenuStore {
+ constructor() {
+ this.menu = Menu.menu;
+ this.eventEmitter = new EventEmitter();
+ }
+
+ getMenu() {
+ return this.menu;
+ }
+
+ addChangeMenuListener(callback) {
+ this.eventEmitter.on('change', callback);
+ }
+
+ removeChangeMenuListener(callback) {
+ this.eventEmitter.removeListener('change', callback);
+ }
+
+ notifyListeners() {
+ this.eventEmitter.emit('change');
+ }
+
+}
+
+const mm = new MainMenuStore();
+
+SpotDispatcher.register(function (action) {
+ switch (action.actionType) {
+ case SpotConstants.RELOAD_MENU:
+ mm.notifyListeners();
+ break;
+ }
+});
+
+module.exports = mm;
[06/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/DendrogramStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/DendrogramStore.js b/spot-oa/ui/dns/js/stores/DendrogramStore.js
index 18c8777..54822c0 100755
--- a/spot-oa/ui/dns/js/stores/DendrogramStore.js
+++ b/spot-oa/ui/dns/js/stores/DendrogramStore.js
@@ -1,50 +1,81 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var DnsConstants = require('../constants/DnsConstants');
-var RestStore = require('../../../js/stores/RestStore');
-var assign = require('object-assign');
-
-var SRC_IP_FILTER = 'ip_dst';
-
-var DendrogramStore = assign(new RestStore(DnsConstants.API_VISUAL_DETAILS), {
- setDate: function (date)
- {
- this.setEndpoint(DnsConstants.API_VISUAL_DETAILS.replace('${date}', date.replace(/-/g, '')));
- },
- setSrcIp: function (srcIp)
- {
- this.setRestFilter(SRC_IP_FILTER, srcIp);
- },
- getSrcIp: function ()
- {
- return this.getRestFilter(SRC_IP_FILTER);
- },
- setData: function (data)
- {
- this._data = data;
-
- this.emitChangeData();
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date'
+const CLIENT_IP_VAR = 'clientIp';
+
+class DendrogramStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$clientIp:SpotIpType!) {
+ dns {
+ ipDetails(date:$date, clientIp:$clientIp) {
+ dns_qry_name: dnsQuery
+ dns_a: dnsQueryAnswers
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.dns.ipDetails;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setClientIp (clientIp) {
+ this.setVariable(CLIENT_IP_VAR, clientIp);
+ }
+
+ unsetClientIp (clientIp) {
+ this.unsetVariable(CLIENT_IP_VAR);
+ }
+
+ getClientIp() {
+ return this.getVariable(CLIENT_IP_VAR);
+ }
+}
+
+const ds = new DendrogramStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- DendrogramStore.setDate(action.date);
+ ds.setDate(action.date);
break;
case SpotConstants.SELECT_IP:
- DendrogramStore.setSrcIp(action.ip);
+ ds.setClientIp(action.ip);
break;
case SpotConstants.RELOAD_SUSPICIOUS:
- DendrogramStore.setSrcIp('');
- DendrogramStore.resetData();
+ ds.unsetClientIp();
+ ds.resetData();
break;
case SpotConstants.RELOAD_DETAILS_VISUAL:
- DendrogramStore.reload();
+ ds.sendQuery();
break;
}
});
-module.exports = DendrogramStore;
+module.exports = ds;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/DetailsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/DetailsStore.js b/spot-oa/ui/dns/js/stores/DetailsStore.js
index 1e8b9b2..e82e70c 100755
--- a/spot-oa/ui/dns/js/stores/DetailsStore.js
+++ b/spot-oa/ui/dns/js/stores/DetailsStore.js
@@ -1,59 +1,94 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var DnsConstants = require('../constants/DnsConstants');
-var RestStore = require('../../../js/stores/RestStore');
-var assign = require('object-assign');
-
-var DNS_SERVER_FILTER = 'dns_qry_name';
-var TIME_FILTER = 'time';
-
-var DetailsStore = assign(new RestStore(DnsConstants.API_DETAILS), {
- errorMessages: {
- 404: 'No details available'
- },
- headers: {
- frame_time: 'Timestamp',
- frame_len: 'Length',
- ip_dst: 'Client IP',
- ip_src: 'Server IP',
- dns_qry_name: 'Query',
- dns_qry_class_name: 'Query Class',
- dns_qry_type_name: 'Query Type',
- dns_qry_rcode_name: 'Response Code',
- dns_a: 'Answer'
- },
- setDate: function (date)
- {
- this.setEndpoint(DnsConstants.API_DETAILS.replace('${date}', date.replace(/-/g, '')));
- },
- setDnsServer: function (dnsServer)
- {
- this.setRestFilter(DNS_SERVER_FILTER, dnsServer);
- },
- setTime: function (time)
- {
- this.setRestFilter(TIME_FILTER, time);
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const DNS_QUERY_VAR = 'dnsQuery';
+const TIME_VAR = 'frameTime';
+
+class DetailsStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+
+ this.headers = {
+ frame_time: 'Timestamp',
+ frame_len: 'Length',
+ ip_dst: 'Client IP',
+ ip_src: 'Server IP',
+ dns_qry_name: 'Query',
+ dns_qry_class_name: 'Query Class',
+ dns_qry_type_name: 'Query Type',
+ dns_qry_rcode_name: 'Response Code',
+ dns_a: 'Answer'
+ };
+
+ this.ITERATOR = ['frame_time', 'frame_len', 'ip_dst', 'ip_src', 'dns_qry_name', 'dns_qry_class_name', 'dns_qry_type_name', 'dns_qry_rcode_name', 'dns_a'];
+ }
+
+ getQuery() {
+ return `
+ query($frameTime:SpotDatetimeType!,$dnsQuery:String!) {
+ dns {
+ edgeDetails(frameTime: $frameTime, dnsQuery:$dnsQuery) {
+ dns_a: dnsQueryAnswers
+ frame_len: frameLength
+ dns_qry_type_name: dnsQueryTypeLabel
+ dns_qry_rcode_name: dnsQueryRcodeLabel
+ ip_dst: clientIp
+ dns_qry_class_name: dnsQueryClassLabel
+ frame_time: frameTime
+ ip_src: serverIp
+ dns_qry_name: dnsQuery
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.dns.edgeDetails;
+ }
+
+ setDnsQuery(dnsQuery) {
+ this.setVariable(DNS_QUERY_VAR, dnsQuery);
+ }
+
+ setTime(time) {
+ this.setVariable(TIME_VAR, time);
+ }
+}
+
+const ds = new DetailsStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
- case SpotConstants.UPDATE_DATE:
- DetailsStore.setDate(action.date);
- break;
case SpotConstants.SELECT_THREAT:
- DetailsStore.setDnsServer(action.threat.dns_qry_name);
- DetailsStore.setTime(action.threat.hh+':00');
+ ds.setDnsQuery(action.threat.dns_qry_name);
+ ds.setTime(action.threat.frame_time);
break;
case SpotConstants.RELOAD_SUSPICIOUS:
- DetailsStore.resetData();
+ ds.resetData();
break;
case SpotConstants.RELOAD_DETAILS:
- DetailsStore.reload();
+ ds.sendQuery();
break;
}
});
-module.exports = DetailsStore;
+module.exports = ds;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/IncidentProgressionStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/IncidentProgressionStore.js b/spot-oa/ui/dns/js/stores/IncidentProgressionStore.js
index 518fa90..308a43e 100755
--- a/spot-oa/ui/dns/js/stores/IncidentProgressionStore.js
+++ b/spot-oa/ui/dns/js/stores/IncidentProgressionStore.js
@@ -1,76 +1,106 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var DnsConstants = require('../constants/DnsConstants');
-var RestStore = require('../../../js/stores/RestStore');
-
-var fields = ['title', 'summary'];
-var filterName;
-
-var IncidentProgressionStore = assign(new RestStore(DnsConstants.API_INCIDENT_PROGRESSION), {
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(DnsConstants.API_INCIDENT_PROGRESSION.replace('${date}', date.replace(/-/g, '')));
- },
- setFilter: function (name, value)
- {
- filterName = name;
- this.setRestFilter('id', value);
- },
- getFilterName: function ()
- {
- return filterName;
- },
- getFilterValue: function ()
- {
- return this.getRestFilter('id');
- },
- clearFilter: function ()
- {
- this.removeRestFilter('id');
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
-SpotDispatcher.register(function (action) {
- switch (action.actionType) {
- case SpotConstants.UPDATE_DATE:
- IncidentProgressionStore.setDate(action.date);
- IncidentProgressionStore.clearFilter();
- IncidentProgressionStore.resetData();
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
- break;
- case SpotConstants.SELECT_COMMENT:
- var comment, filterParts, key;
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+const QUERY_VAR = 'dnsQuery';
+const CLIENT_IP_VAR = 'clientIp';
+
+class IncidentProgressionStore extends ObservableGraphQLStore {
+ constructor() {
+ super();
+
+ this.filterName = null;
+ }
+
+ getQuery() {
+ return `
+ query($date:SpotDateType, $dnsQuery: String, $clientIp:SpotIpType) {
+ dns {
+ threat {
+ incidentProgression(date:$date, dnsQuery:$dnsQuery, clientIp:$clientIp) {
+ ...QueryFragment
+ ...ClientIpFragment
+ }
+ }
+ }
+ }
- IncidentProgressionStore.clearFilter();
+ fragment QueryFragment on DnsIncidentProgressionQueryType {
+ dnsQuery
+ }
- comment = action.comment;
+ fragment ClientIpFragment on DnsIncidentProgressionClientIpType {
+ clientIp
+ }
+ `;
+ }
- filterParts = [];
+ unboxData(data) {
+ return data.dns.threat.incidentProgression;
+ }
- for (key in comment)
- {
- // Skip comment fields
- if (fields.indexOf(key)>=0) continue;
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
- if (comment[key])
- {
- IncidentProgressionStore.setFilter(key, comment[key]);
+ setFilter(name, value) {
+ this.filterName = name==QUERY_VAR?name:CLIENT_IP_VAR;
+ this.setVariable(this.filterName, value);
+ }
+
+ getFilterName() {
+ return this.filterName;
+ }
+
+ getFilterValue() {
+ return this.getVariable(this.filterName);
+ }
+
+ clearFilter() {
+ this.unsetVariable(this.filterName);
+ this.filterName = null;
+ }
+}
+
+const ips = new IncidentProgressionStore();
+
+SpotDispatcher.register(function (action) {
+ switch (action.actionType) {
+ case SpotConstants.UPDATE_DATE:
+ ips.setDate(action.date);
+ ips.clearFilter();
+ ips.resetData();
+
+ break;
+ case SpotConstants.SELECT_COMMENT:
+ ips.clearFilter();
- break;
- }
- }
+ let filterName = QUERY_VAR in action.comment ? QUERY_VAR : CLIENT_IP_VAR;
+ ips.setFilter(filterName, action.comment[filterName]);
- IncidentProgressionStore.reload();
+ ips.sendQuery();
break;
}
});
-module.exports = IncidentProgressionStore;
+module.exports = ips;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/NotebookStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/NotebookStore.js b/spot-oa/ui/dns/js/stores/NotebookStore.js
new file mode 100644
index 0000000..8b87047
--- /dev/null
+++ b/spot-oa/ui/dns/js/stores/NotebookStore.js
@@ -0,0 +1,107 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SuspiciousStore = require('./SuspiciousStore');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const SCORED_ELEMENTS = 'input';
+const RESET_SCORED_ELEMENTS = 'date';
+
+class NotebookStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+ this.selectedQuery = 'scoreQuery';
+ this.query = {
+ scoreQuery: `
+ mutation($input:[DnsScoreType!]!) {
+ dns{
+ score(input:$input)
+ {success}
+ }
+ }`,
+ resetQuery: `
+ mutation($date:SpotDateType!) {
+ dns{
+ resetScoredConnections(date:$date){
+ success
+ }
+ }
+ }`
+ };
+ this.completeClass = false;
+ }
+
+ getQuery() {
+ return this.query[this.selectedQuery];
+ }
+
+ unboxData(data) {
+ return data.dns.suspicious;
+ }
+
+ setScoredElements(scoredElements) {
+ this.selectedQuery = 'scoreQuery';
+ this.setVariable(SCORED_ELEMENTS, scoredElements);
+ this.unsetVariables(RESET_SCORED_ELEMENTS);
+ }
+
+ resetScoredElements(date) {
+ this.selectedQuery = 'resetQuery';
+ this.setVariable(RESET_SCORED_ELEMENTS, date);
+ this.unsetVariables(SCORED_ELEMENTS);
+ }
+
+ unsetVariables(variable) {
+ this.unsetVariable(variable);
+ }
+
+ reloadElements() {
+ SuspiciousStore.sendQuery();
+ }
+
+ changeCssCls() {
+ this.completeClass = !this.completeClass;
+ }
+
+
+}
+
+const ns = new NotebookStore();
+
+SpotDispatcher.register(function (action) {
+ switch (action.actionType) {
+ case SpotConstants.SAVE_SCORED_ELEMENTS:
+ ns.setScoredElements(action.scoredElems);
+ ns.sendQuery();
+ break;
+ case SpotConstants.RESET_SCORED_ELEMENTS:
+ ns.resetScoredElements(action.date);
+ ns.sendQuery();
+ break;
+ case SpotConstants.RELOAD_SUSPICIOUS:
+ ns.resetData();
+ break;
+ case SpotConstants.CHANGE_CSS_CLS:
+ ns.changeCssCls();
+ break;
+ }
+});
+
+module.exports = ns;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/SuspiciousStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/SuspiciousStore.js b/spot-oa/ui/dns/js/stores/SuspiciousStore.js
index e3e4e47..21b728d 100755
--- a/spot-oa/ui/dns/js/stores/SuspiciousStore.js
+++ b/spot-oa/ui/dns/js/stores/SuspiciousStore.js
@@ -1,203 +1,191 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var DnsConstants = require('../constants/DnsConstants');
-var RestStore = require('../../../js/stores/RestStore');
-var SpotUtils = require('../../../js/utils/SpotUtils');
-var assign = require('object-assign');
-
-var IP_FILTER = 'ip_dst';
-var DNS_FILTER = 'dns_qry_name';
-
-var CHANGE_FILTER_EVENT = 'change_filter';
-var CHANGE_DATE_EVENT = 'change_date';
-var HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
-var UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
-var SELECT_THREAT_EVENT = 'select_treath';
-
-var filter = '';
-var filterName = '';
-var date = '';
-var highlightedThread = null;
-var selectedThread = null;
-var unfilteredData = null;
-
-var SuspiciousStore = assign(new RestStore(DnsConstants.API_SUSPICIOUS), {
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- headers: {
- frame_time: 'Timestamp',
- ip_dst: 'Client IP',
- dns_qry_name: 'Query',
- query_rep: ' ',
- dns_qry_class_name: 'Query Class',
- dns_qry_type_name: 'Query Type',
- dns_qry_rcode_name: 'Response Code'
- },
- ITERATOR: ['frame_time', 'ip_dst', 'dns_qry_name', 'query_rep', 'dns_qry_class_name', 'dns_qry_type_name', 'dns_qry_rcode_name'],
- getData: function ()
- {
- var state;
-
- if (!filter || !unfilteredData)
- {
- state = this._data;
- }
- else
- {
- state = assign(
- {},
- unfilteredData
- );
-
- if (unfilteredData.data) {
- state.data = unfilteredData.data.filter(function (item) {
- return filterName===IP_FILTER ? item[filterName]==filter : item[filterName].indexOf(filter)>=0;
- });
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SpotUtils = require('../../../js/utils/SpotUtils');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const DATE_VAR = 'date';
+const IP_VAR = 'ipClient';
+const DNS_VAR = 'dnsQuery';
+
+const CHANGE_FILTER_EVENT = 'change_filter';
+const HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
+const UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
+const SELECT_THREAT_EVENT = 'select_treath';
+
+class SuspiciousStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+
+ this.headers = {
+ frame_time: 'Timestamp',
+ ip_dst: 'Client IP',
+ dns_qry_name: 'Query',
+ query_rep: ' ',
+ dns_qry_class_name: 'Query Class',
+ dns_qry_type_name: 'Query Type',
+ dns_qry_rcode_name: 'Response Code'
+ };
+
+ this.ITERATOR = ['frame_time', 'ip_dst', 'dns_qry_name', 'query_rep', 'dns_qry_class_name', 'dns_qry_type_name', 'dns_qry_rcode_name'];
+
+ this.highlightedThread = null;
+ this.selectedThread = null;
+ }
+
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$dnsQuery:String,$ipClient:SpotIpType) {
+ dns {
+ suspicious(date: $date, dnsQuery:$dnsQuery, clientIp:$ipClient) {
+ unix_tstamp: unixTimestamp
+ dns_qry_type: dnsQueryType
+ frame_len: frameLength
+ dns_qry_type_name: dnsQueryTypeLabel
+ dns_sev: dnsQuerySev
+ ip_sev: clientIpSev
+ frame_time: frameTime
+ dns_qry_class_name: dnsQueryClassLabel
+ dns_qry_rcode: dnsQueryRcode
+ score
+ dns_qry_rcode_name: dnsQueryRcodeLabel
+ network_context: networkContext
+ tld
+ dns_qry_class: dnsQueryClass
+ ip_dst: clientIp
+ query_rep: dnsQueryRep
+ dns_qry_name: dnsQuery
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.dns.suspicious;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setFilter(filter) {
+ if (filter==='') {
+ this.unsetVariable(IP_VAR);
+ this.unsetVariable(DNS_VAR);
}
+ else if (SpotUtils.IP_V4_REGEX.test(filter)) {
+ this.unsetVariable(DNS_VAR);
+ this.setVariable(IP_VAR, filter);
+ }
+ else {
+ this.unsetVariable(IP_VAR);
+ this.setVariable(DNS_VAR, filter);
+ }
+
+ this.notifyListeners(CHANGE_FILTER_EVENT);
}
- if (state.data) {
- state.data = state.data.filter(function (item) {
- return item.dns_sev=="0";
- });
-
- if (state.data.length>SpotConstants.MAX_SUSPICIOUS_ROWS) state.data = state.data.slice(0, SpotConstants.MAX_SUSPICIOUS_ROWS);
- }
-
- return state;
- },
- setData: function (data)
- {
- this._data = unfilteredData = data;
-
- this.emitChangeData();
- },
- setDate: function (date)
- {
- this.setEndpoint(DnsConstants.API_SUSPICIOUS.replace('${date}', date.replace(/-/g, '')));
- },
- setFilter: function (newFilter)
- {
- filter = newFilter;
-
- if (filter==='')
- {
- filterName = '';
- this.removeRestFilter(IP_FILTER);
- this.removeRestFilter(DNS_FILTER);
- }
- else if (SpotUtils.IP_V4_REGEX.test(filter))
- {
- this.removeRestFilter(DNS_FILTER);
- this.setRestFilter(IP_FILTER, filter);
- filterName = IP_FILTER;
- }
- else
- {
- this.removeRestFilter(IP_FILTER);
- this.setRestFilter(DNS_FILTER, filter);
- filterName = DNS_FILTER;
- }
-
- this.emitChangeFilter();
- },
- getFilter: function (){
- return filter;
- },
- emitChangeFilter: function () {
- this.emit(CHANGE_FILTER_EVENT);
- },
- addChangeFilterListener: function (callback) {
- this.on(CHANGE_FILTER_EVENT, callback);
- },
- removeChangeFilterListener: function (callback) {
- this.removeListener(CHANGE_FILTER_EVENT, callback);
- },
- highlightThreat: function (threat)
- {
- highlightedThread = threat;
- this.emitHighlightThreat();
- },
- getHighlightedThreat: function ()
- {
- return highlightedThread;
- },
- addThreatHighlightListener: function (callback)
- {
- this.on(HIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatHighlightListener: function (callback)
- {
- this.removeListener(HIGHLIGHT_THREAT_EVENT, callback);
- },
- emitHighlightThreat: function ()
- {
- this.emit(HIGHLIGHT_THREAT_EVENT);
- },
- unhighlightThreat: function ()
- {
- highlightedThread = null;
- this.emitUnhighlightThreat();
- },
- addThreatUnhighlightListener: function (callback)
- {
- this.on(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatUnhighlightListener: function (callback)
- {
- this.removeListener(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- emitUnhighlightThreat: function ()
- {
- this.emit(UNHIGHLIGHT_THREAT_EVENT);
- },
- selectThreat: function (threat)
- {
- selectedThread = threat;
- this.emitThreatSelect();
- },
- getSelectedThreat: function ()
- {
- return selectedThread;
- },
- addThreatSelectListener: function (callback)
- {
- this.on(SELECT_THREAT_EVENT, callback);
- },
- removeThreatSelectListener: function (callback)
- {
- this.removeListener(SELECT_THREAT_EVENT, callback);
- },
- emitThreatSelect: function ()
- {
- this.emit(SELECT_THREAT_EVENT);
- }
-});
+ getFilter() {
+ return this.getVariable(IP_VAR) || this.getVariable(DNS_VAR) || '';
+ }
+
+ addChangeFilterListener(callback) {
+ this.addListener(CHANGE_FILTER_EVENT, callback);
+ }
+
+ removeChangeFilterListener(callback) {
+ this.removeListener(CHANGE_FILTER_EVENT, callback);
+ }
+
+ highlightThreat(threat) {
+ this.highlightedThread = threat;
+ this.notifyListeners(HIGHLIGHT_THREAT_EVENT);
+ }
+
+ getHighlightedThreat() {
+ return this.highlightedThread;
+ }
+
+ unhighlightThreat() {
+ this.highlightedThread = null;
+ this.notifyListeners(UNHIGHLIGHT_THREAT_EVENT);
+ }
+
+ addThreatHighlightListener(callback) {
+ this.addListener(HIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatHighlightListener(callback) {
+ this.removeListener(HIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ addThreatUnhighlightListener(callback) {
+ this.addListener(UNHIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatUnhighlightListener(callback) {
+ this.removeListener(UNHIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ selectThreat(threat) {
+ this.selectedThread = threat;
+ this.notifyListeners(SELECT_THREAT_EVENT);
+ }
+
+ getSelectedThreat() {
+ return this.selectedThread;
+ }
+
+ addThreatSelectListener(callback) {
+ this.addListener(SELECT_THREAT_EVENT, callback);
+ }
+
+ removeThreatSelectListener(callback) {
+ this.removeListener(SELECT_THREAT_EVENT, callback);
+ }
+}
+
+const ss = new SuspiciousStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_FILTER:
- SuspiciousStore.setFilter(action.filter);
+ ss.setFilter(action.filter);
break;
case SpotConstants.UPDATE_DATE:
- SuspiciousStore.setDate(action.date);
+ ss.setDate(action.date);
break;
case SpotConstants.RELOAD_SUSPICIOUS:
- SuspiciousStore.reload();
+ ss.sendQuery();
break;
case SpotConstants.HIGHLIGHT_THREAT:
- SuspiciousStore.highlightThreat(action.threat);
+ ss.highlightThreat(action.threat);
break;
case SpotConstants.UNHIGHLIGHT_THREAT:
- SuspiciousStore.unhighlightThreat();
+ ss.unhighlightThreat();
break;
case SpotConstants.SELECT_THREAT:
- SuspiciousStore.selectThreat(action.threat);
+ ss.selectThreat(action.threat);
break;
}
});
-module.exports = SuspiciousStore;
+module.exports = ss;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/storyboard.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/storyboard.js b/spot-oa/ui/dns/js/storyboard.js
index 7374828..6af2e8d 100755
--- a/spot-oa/ui/dns/js/storyboard.js
+++ b/spot-oa/ui/dns/js/storyboard.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -8,6 +23,12 @@ const SpotConstants = require('../../js/constants/SpotConstants');
const SpotUtils = require('../../js/utils/SpotUtils');
const DateInput = require('../../js/components/DateInput.react');
const StoryboardActions = require('../../js/actions/StoryboardActions');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
ReactDOM.render(
(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/suspicious.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/suspicious.js b/spot-oa/ui/dns/js/suspicious.js
index e33dd91..e5d4c06 100755
--- a/spot-oa/ui/dns/js/suspicious.js
+++ b/spot-oa/ui/dns/js/suspicious.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -11,6 +26,7 @@ const SpotUtils = require('../../js/utils/SpotUtils');
// Build and Render Toolbar
const FilterInput = require('./components/FilterInput.react');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
function resetFilterAndReload()
{
@@ -18,6 +34,15 @@ function resetFilterAndReload()
EdInActions.reloadSuspicious();
};
+function switchComponents () {
+ SpotUtils.switchDivs(SpotConstants.DETAILS_PANEL, SpotConstants.SCORING_PANEL);
+};
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
ReactDOM.render(
(
<form className="form-inline">
@@ -54,11 +79,9 @@ const Panel = require('../../js/components/Panel.react');
const SuspiciousPanel = require('./components/SuspiciousPanel.react');
const NetworkViewPanel = require('./components/NetworkViewPanel.react');
-const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+const ScoreNotebook = require('./components/ScoreNotebook.react');
const DetailsPanel = require('./components/DetailsPanel.react');
-const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure(SpotConstants.NOTEBOOK_PANEL);
-
ReactDOM.render(
<div id="spot-content">
<PanelRow>
@@ -69,14 +92,18 @@ ReactDOM.render(
<NetworkViewPanel />
</Panel>
</PanelRow>
- <PanelRow>
- <Panel title={ipynbClosure.getTitle()} container extraButtons={ipynbClosure.getButtons}>
- <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="dns/${date}/Edge_Investigation.ipynb" />
- </Panel>
- <Panel title={SpotConstants.DETAILS_PANEL} container expandable>
- <DetailsPanel title={SpotConstants.DETAILS_PANEL} />
- </Panel>
- </PanelRow>
+ <div className="sortable">
+ <PanelRow title={SpotConstants.SCORING_PANEL}>
+ <Panel title={SpotConstants.SCORING_PANEL} reloadable switchable onReload={EdInActions.reloadSuspicious} onSwitch={switchComponents} className="col-md-12">
+ <ScoreNotebook />
+ </Panel>
+ </PanelRow>
+ <PanelRow title={SpotConstants.DETAILS_PANEL}>
+ <Panel title={SpotConstants.DETAILS_PANEL} container switchable expandable onSwitch={switchComponents} className="col-md-12">
+ <DetailsPanel title={SpotConstants.DETAILS_PANEL} />
+ </Panel>
+ </PanelRow>
+ </div>
</div>,
document.getElementById('spot-content-wrapper')
);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/threat-investigation.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/threat-investigation.js b/spot-oa/ui/dns/js/threat-investigation.js
index efd5c1c..71998ef 100755
--- a/spot-oa/ui/dns/js/threat-investigation.js
+++ b/spot-oa/ui/dns/js/threat-investigation.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -7,6 +22,12 @@ const SpotActions = require('../../js/actions/SpotActions');
const SpotUtils = require('../../js/utils/SpotUtils');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
ReactDOM.render(
(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/package.json b/spot-oa/ui/dns/package.json
index d3f31bb..f57d258 100755
--- a/spot-oa/ui/dns/package.json
+++ b/spot-oa/ui/dns/package.json
@@ -6,10 +6,12 @@
"watch-suspicious": "watchify js/suspicious.js -o js/suspicious.bundle.min.js -v -d",
"watch-threat-investigation": "watchify js/threat-investigation.js -o js/threat-investigation.bundle.min.js -v -d",
"watch-storyboard": "watchify js/storyboard.js -o js/storyboard.bundle.min.js -v -d",
- "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard",
+ "watch-notebooks": "watchify js/notebooks.js -o js/notebooks.bundle.min.js -v -d",
+ "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard && npm run build-notebooks",
"build-suspicious": "browserify js/suspicious.js | uglifyjs -cm > js/suspicious.bundle.min.js",
"build-threat-investigation": "browserify js/threat-investigation.js | uglifyjs -cm > js/threat-investigation.bundle.min.js",
- "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js"
+ "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js",
+ "build-notebooks": "browserify js/notebooks.js | uglifyjs -cm > js/notebooks.bundle.min.js"
},
"browserify": {
"transform": [
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/storyboard.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/storyboard.html b/spot-oa/ui/dns/storyboard.html
index fdaee24..46312e0 100755
--- a/spot-oa/ui/dns/storyboard.html
+++ b/spot-oa/ui/dns/storyboard.html
@@ -21,6 +21,7 @@
<title>DNS :: Storyboard</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -70,62 +71,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../proxy/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../proxy/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../proxy/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/suspicious.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/suspicious.html b/spot-oa/ui/dns/suspicious.html
index 18c85b2..0c784be 100755
--- a/spot-oa/ui/dns/suspicious.html
+++ b/spot-oa/ui/dns/suspicious.html
@@ -21,6 +21,7 @@
<title>DNS :: Suspicious</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -32,6 +33,8 @@
<link rel="stylesheet" type="text/css" href="../css/main.css" />
<!-- Dendrogram styles -->
<link rel="stylesheet" type="text/css" href="../css/dendrogram.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
<!-- Favicon -->
<link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
@@ -75,62 +78,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../proxy/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../proxy/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../proxy/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
@@ -154,6 +102,7 @@
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
<script type="application/javascript" src="../js/tooltip.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
<script type="application/javascript" src="js/suspicious.bundle.min.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/threat-investigation.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/threat-investigation.html b/spot-oa/ui/dns/threat-investigation.html
index f7f66d0..1c504bf 100755
--- a/spot-oa/ui/dns/threat-investigation.html
+++ b/spot-oa/ui/dns/threat-investigation.html
@@ -21,6 +21,7 @@
<title>DNS :: Threat Investigation</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -68,62 +69,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../proxy/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../proxy/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../proxy/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/ipython_notebook.html b/spot-oa/ui/flow/ipython_notebook.html
new file mode 100644
index 0000000..17c59c8
--- /dev/null
+++ b/spot-oa/ui/flow/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>Netflow :: Advanced mode</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Font awesome ICONS-->
+ <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+
+ <style>
+ .spot-row {
+ height: 100%;
+ }
+
+ td.srcIP_rep, td.dstIP_rep {
+ white-space: nowrap;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: Netflow :: Advanced mode</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+ <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/DetailsChordsPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/DetailsChordsPanel.react.js b/spot-oa/ui/flow/js/components/DetailsChordsPanel.react.js
index 2767274..c88127d 100755
--- a/spot-oa/ui/flow/js/components/DetailsChordsPanel.react.js
+++ b/spot-oa/ui/flow/js/components/DetailsChordsPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/DetailsPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/DetailsPanel.react.js b/spot-oa/ui/flow/js/components/DetailsPanel.react.js
index 538a3dd..9263697 100755
--- a/spot-oa/ui/flow/js/components/DetailsPanel.react.js
+++ b/spot-oa/ui/flow/js/components/DetailsPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/DetailsTablePanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/DetailsTablePanel.react.js b/spot-oa/ui/flow/js/components/DetailsTablePanel.react.js
index ed222e7..ac078dc 100755
--- a/spot-oa/ui/flow/js/components/DetailsTablePanel.react.js
+++ b/spot-oa/ui/flow/js/components/DetailsTablePanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/FilterInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/FilterInput.react.js b/spot-oa/ui/flow/js/components/FilterInput.react.js
index 035547d..ae30bbf 100755
--- a/spot-oa/ui/flow/js/components/FilterInput.react.js
+++ b/spot-oa/ui/flow/js/components/FilterInput.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/GlobeViewPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/GlobeViewPanel.react.js b/spot-oa/ui/flow/js/components/GlobeViewPanel.react.js
index 7d3c4ea..23e4802 100755
--- a/spot-oa/ui/flow/js/components/GlobeViewPanel.react.js
+++ b/spot-oa/ui/flow/js/components/GlobeViewPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const $ = require('jquery');
const d3 = require('d3');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/ImpactAnalysisPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/ImpactAnalysisPanel.react.js b/spot-oa/ui/flow/js/components/ImpactAnalysisPanel.react.js
index a84e9ec..dc3f1c2 100755
--- a/spot-oa/ui/flow/js/components/ImpactAnalysisPanel.react.js
+++ b/spot-oa/ui/flow/js/components/ImpactAnalysisPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/IncidentProgressionPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/IncidentProgressionPanel.react.js b/spot-oa/ui/flow/js/components/IncidentProgressionPanel.react.js
index 78112d3..1c53278 100755
--- a/spot-oa/ui/flow/js/components/IncidentProgressionPanel.react.js
+++ b/spot-oa/ui/flow/js/components/IncidentProgressionPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/NetworkViewPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/NetworkViewPanel.react.js b/spot-oa/ui/flow/js/components/NetworkViewPanel.react.js
index db943ee..66a6bae 100755
--- a/spot-oa/ui/flow/js/components/NetworkViewPanel.react.js
+++ b/spot-oa/ui/flow/js/components/NetworkViewPanel.react.js
@@ -1,5 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ContentLoaderMixin = require('../../../js/components/ContentLoaderMixin.react');
@@ -49,7 +63,7 @@ function getNodesFromData(data) {
nodes[id] = {
id: id,
label: item[field],
- internalIp: item[mapper[field]]==='1',
+ internalIp: item[mapper[field]],
hits: 1
};
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/ScoreNotebook.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/ScoreNotebook.react.js b/spot-oa/ui/flow/js/components/ScoreNotebook.react.js
new file mode 100644
index 0000000..22eeec7
--- /dev/null
+++ b/spot-oa/ui/flow/js/components/ScoreNotebook.react.js
@@ -0,0 +1,221 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var SuspiciousStore = require('../stores/SuspiciousStore');
+var NotebookStore = require('../stores/NotebookStore');
+var EdInActions = require('../../../js/actions/EdInActions');
+var SelectInput = require('../../../js/components/scoring/SelectInput.react');
+var FilterSelectInput = require('../../../js/components/scoring/FilterSelectInput.react');
+var ButtonsScoring = require('../../../js/components/scoring/ButtonsScoring.react');
+var ScoreMessage = require('../../../js/components/scoring/ScoreMessage.react');
+var RatingInput = require('../../../js/components/scoring/RatingInput.react');
+var SearchGlobalInput = require('../../../js/components/scoring/SearchGlobalInput.react');
+
+var SpotUtils = require('../../../js/utils/SpotUtils');
+
+var ScoreNotebook = React.createClass({
+ // mixins: [GridPanelMixin],
+ emptySetMessage: 'There is no data available for selected date.',
+ propTypes: {
+ date: React.PropTypes.string.isRequired,
+ },
+ getInitialState: function () {
+ return {
+ scoredEmelents: []
+ };
+ },
+ componentDidMount: function() {
+ SuspiciousStore.addChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ this.setState({size: NotebookStore.completeClass})
+ },
+ componentWillUnmount: function () {
+ SuspiciousStore.removeChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ },
+ render: function () {
+ var content, state, data, cssCls, srcIpArr = [], dstIpArr = [], srcPortArr = [], dstPortArr = [];
+ state = this.state || {};
+
+ if (state.error)
+ {
+ content = (
+ <div className="text-center text-danger">
+ {state.error}
+ </div>
+ );
+ }
+ else if (state.loading)
+ {
+ content = (
+ <div className="spot-loader">
+ Loading <span className="spinner"></span>
+ </div>
+ );
+ }
+ else if (!state.data || state.data.length === 0)
+ {
+ content = (
+ <div className="text-center">
+ {this.emptySetMessage || ''}
+ </div>
+ );
+ }
+ else
+ {
+ state.data.map((obj) => {
+ if(srcIpArr.indexOf(obj.srcIP) === -1) {
+ srcIpArr.push(obj.srcIP);
+ }
+ if(dstIpArr.indexOf(obj.dstIP) === -1) {
+ dstIpArr.push(obj.dstIP);
+ }
+ if(srcPortArr.indexOf(obj.sport) === -1) {
+ srcPortArr.push(obj.sport);
+ }
+ if(dstPortArr.indexOf(obj.dport) === -1) {
+ dstPortArr.push(obj.dport);
+ }
+ });
+
+ data = [
+ {value: 1, name: 'High', radioName: 'optradio', selected: true},
+ {value: 2, name: 'Medium', radioName: 'optradio', selected: false},
+ {value: 3, name: 'Low', radioName: 'optradio', selected: false}
+ ];
+
+ content = (
+ <div>
+ <div className="margin-up-down">
+ <SearchGlobalInput col="6"/>
+ <RatingInput data={data} col="6"/>
+ </div>
+ <div className="margin-up-down">
+ <ButtonsScoring name="Score" action="score" onChange={this.score.bind(this)} col="3"/>
+ <ButtonsScoring name="Save" action="save" onChange={this.save.bind(this)} col="3"/>
+ <ButtonsScoring name="Reset Scoring" action="reset" onChange={this.reset.bind(this)} col="3"/>
+ </div>
+ <div className="margin-up-down">
+ <FilterSelectInput nameBox="Source Ip" idSelect="#srcIp" idInput="srcIpIn" col="3"/>
+ <FilterSelectInput nameBox="Dest IP" idSelect="#dstIp" idInput="dstIpIn" col="3"/>
+ <FilterSelectInput nameBox="Src Port" idSelect="#srcPort" idInput="srcPortIn" col="3"/>
+ <FilterSelectInput nameBox="Dst Port" idSelect="#dstPort" idInput="dstPortIn" col="3"/>
+ </div>
+ <div className="margin-up-down">
+ <SelectInput title="Source IP" who="srcIp" options={srcIpArr} col="3"/>
+ <SelectInput title="Dest IP" who="dstIp" options={dstIpArr} col="3"/>
+ <SelectInput title="Src Port" who="srcPort" options={srcPortArr} col="3"/>
+ <SelectInput title="Dst Port" who="dstPort" options={dstPortArr} col="3"/>
+ </div>
+ <div className="margin-up-down">
+ <ScoreMessage who="scoreMsg"/>
+ </div>
+ </div>
+ );
+ }
+ cssCls = this.state.size ? 'col-md-6 col-lg-6 col-xs-12' : 'col-md-offset-3 col-lg-offset-3 col-md-6 col-lg-6 col-xs-12';
+
+ return(
+ <div className={cssCls + ' spot-frame'}>
+ {content}
+ </div>
+ )
+ },
+ _onChange: function() {
+ const data = SuspiciousStore.getData();
+ this.setState(data);
+ },
+ reset: function() {
+ swal({
+ title: 'Are you sure?',
+ text: "You won't be able to revert this!",
+ type: 'warning',
+ showCancelButton: true,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Yes, reset all!'
+ }).then(() => {
+ EdInActions.resetScoring(SpotUtils.getCurrentDate());
+ swal({
+ title: 'Done!',
+ text: "All scores have been reset.",
+ type: 'success',
+ showCancelButton: false,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Ok!'
+ }).then(() => {
+ this.setState({loading: true});
+ });
+ });
+ },
+ save: function() {
+ let variables = [];
+
+ if (this.state.scoredEmelents.length === 0) {
+ swal('Warning.','You need to score at least 1 threat.','warning');
+ } else {
+ this.state.scoredEmelents.map((row) => {
+ variables.push({
+ 'date': SpotUtils.getCurrentDate(),
+ 'score': row[4],
+ 'srcIp': row[0] || null,
+ 'dstIp': row[1] || null,
+ 'srcPort': row[2] || null,
+ 'dstPort': row[3] || null
+ });
+ });
+
+ EdInActions.saveScoring(variables);
+ $('#scoreMsg').addClass("hidden");
+ this.setState({scoredEmelents: [], loading: true});
+ }
+ },
+ score: function() {
+ //this should be changed to take all data at the time, time needed.
+ let dataScored = this.state.scoredEmelents || [];
+ let quickIpScoring = document.getElementById('globalTxt').value;
+ let srcIp = document.getElementById('srcIp').value;
+ let dstIp = document.getElementById('dstIp').value;
+ let srcPort = document.getElementById('srcPort').value;
+ let dstPort = document.getElementById('dstPort').value;
+ let rating = $('input[name="optradio"]:checked').val();
+
+ //callback from the father
+ if(quickIpScoring !== '') {
+ dataScored.push([quickIpScoring, dstIp, srcPort, dstPort, rating]);
+ dataScored.push([srcIp, quickIpScoring, srcPort, dstPort, rating]);
+ } else {
+ dataScored.push([srcIp, dstIp, srcPort, dstPort, rating]);
+ }
+
+ $('#scoreMsg').removeClass("hidden");
+ this.removeSelected([quickIpScoring, srcIp, dstIp, srcPort, dstPort]);
+ this.setState({scoredEmelents: dataScored});
+ },
+ removeSelected: function(data) {
+ //when an user score, all elements selected need to be removed.
+ data.map((element) => data.map((e) => e !== '' ? $(`option[value="${e}"]`).remove() : ''));
+ $(".select-picker, #globalTxt").val('');
+ }
+
+});
+
+
+module.exports = ScoreNotebook;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/SuspiciousPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/SuspiciousPanel.react.js b/spot-oa/ui/flow/js/components/SuspiciousPanel.react.js
index 8cf1599..9230243 100755
--- a/spot-oa/ui/flow/js/components/SuspiciousPanel.react.js
+++ b/spot-oa/ui/flow/js/components/SuspiciousPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
@@ -137,8 +152,7 @@ var SuspiciousPanel = React.createClass({
_render_score_cell: false,
_render_srcDomain_cell: false,
_render_srcGeo_cell: false,
- _render_srcIpInternal_cell: false,
- _render_sev_cell: false
+ _render_srcIpInternal_cell: false
});
module.exports = SuspiciousPanel;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/components/TimelinePanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/components/TimelinePanel.react.js b/spot-oa/ui/flow/js/components/TimelinePanel.react.js
index 081bfef..2ae0789 100755
--- a/spot-oa/ui/flow/js/components/TimelinePanel.react.js
+++ b/spot-oa/ui/flow/js/components/TimelinePanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react') ;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/constants/NetflowConstants.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/constants/NetflowConstants.js b/spot-oa/ui/flow/js/constants/NetflowConstants.js
index a3b78c7..84eed16 100755
--- a/spot-oa/ui/flow/js/constants/NetflowConstants.js
+++ b/spot-oa/ui/flow/js/constants/NetflowConstants.js
@@ -1,16 +1,23 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var NetflowConstants = {
// Data source URLS
- API_SUSPICIOUS: '../../data/flow/${date}/flow_scores.csv',
- API_DETAILS: '../../data/flow/${date}/edge-${src_ip}-${dst_ip}-${time}.tsv',
- API_VISUAL_DETAILS: '../../data/flow/${date}/chord-${ip}.tsv',
- API_COMMENTS: '../../data/flow/${date}/threats.csv',
- API_INCIDENT_PROGRESSION: '../../data/flow/${date}/threat-dendro-${ip}.json',
- API_IMPACT_ANALYSIS: '../../data/flow/${date}/stats-${ip}.json',
- API_GLOBE_VIEW: '../../data/flow/${date}/globe-${ip}.json',
- API_WORLD_110M: '../flow/world-110m.json',
- API_TIMELINE: '../../data/flow/${date}/sbdet-${ip}.tsv'
+ API_WORLD_110M: '../flow/world-110m.json'
};
module.exports = NetflowConstants;
[12/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/proxy/query.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/proxy/query.py b/spot-oa/api/graphql/proxy/query.py
new file mode 100644
index 0000000..1e32f34
--- /dev/null
+++ b/spot-oa/api/graphql/proxy/query.py
@@ -0,0 +1,582 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date
+from graphql import (
+ GraphQLObjectType,
+ GraphQLArgument,
+ GraphQLNonNull,
+ GraphQLField,
+ GraphQLString,
+ GraphQLInt,
+ GraphQLList
+)
+
+from api.graphql.common import SpotDateType, SpotDatetimeType, SpotIpType, IngestSummaryType
+import api.resources.proxy as Proxy
+
+SuspiciousType = GraphQLObjectType(
+ name='ProxySuspiciousType',
+ fields={
+ 'datetime': GraphQLField(
+ type=SpotDatetimeType,
+ description='Start time of the request',
+ resolver=lambda root, *_: '{} {}'.format(root.get('tdate') or '1970-01-01', root.get('time') or '00:00:00')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('clientip')
+ ),
+ 'host': GraphQLField(
+ type=GraphQLString,
+ description='Host name from the client request URL',
+ resolver=lambda root, *_: root.get('host')
+ ),
+ 'requestMethod': GraphQLField(
+ type=GraphQLString,
+ description='HTTP request method',
+ resolver=lambda root, *_: root.get('reqmethod')
+ ),
+ 'userAgent': GraphQLField(
+ type=GraphQLString,
+ description='Client\'s user agent',
+ resolver=lambda root, *_: root.get('useragent')
+ ),
+ 'responseContentType': GraphQLField(
+ type=GraphQLString,
+ description='HTTP response content type (MIME)',
+ resolver=lambda root, *_: root.get('resconttype')
+ ),
+ 'duration': GraphQLField(
+ type=GraphQLInt,
+ description='Connection duration',
+ resolver=lambda root, *_: root.get('duration')
+ ),
+ 'username': GraphQLField(
+ type=GraphQLString,
+ description='Username used for authetication',
+ resolver=lambda root, *_: root.get('username')
+ ),
+ 'webCategory': GraphQLField(
+ type=GraphQLString,
+ description='Web content categories',
+ resolver=lambda root, *_: root.get('webcat')
+ ),
+ 'referer': GraphQLField(
+ type=GraphQLString,
+ description='The address of the webpage that linked to the resource being requested',
+ resolver=lambda root, *_: root.get('referer')
+ ),
+ 'responseCode': GraphQLField(
+ type=GraphQLInt,
+ description='HTTP response code',
+ resolver=lambda root, *_: root.get('respcode') or 0
+ ),
+ 'uriPort': GraphQLField(
+ type=GraphQLInt,
+ description='URI port',
+ resolver=lambda root, *_: root.get('uriport')
+ ),
+ 'uriPath': GraphQLField(
+ type=GraphQLString,
+ description='URI path',
+ resolver=lambda root, *_: root.get('uripath')
+ ),
+ 'uriQuery': GraphQLField(
+ type=GraphQLString,
+ description='URI query',
+ resolver=lambda root, *_: root.get('uriquery')
+ ),
+ 'serverIp': GraphQLField(
+ type=SpotIpType,
+ description='Server/Proxy IP',
+ resolver=lambda root, *_: root.get('serverip')
+ ),
+ 'serverToClientBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Number of bytes sent from appliance to client',
+ resolver=lambda root, *_: root.get('scbytes')
+ ),
+ 'clientToServerBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Number of bytes sent from client to appliance',
+ resolver=lambda root, *_: root.get('csbytes')
+ ),
+ 'uri': GraphQLField(
+ type=GraphQLString,
+ description='The original URI requested',
+ resolver=lambda root, *_: root.get('fulluri')
+ ),
+ 'score': GraphQLField(
+ type=GraphQLInt,
+ description='Spot ML score value',
+ resolver=lambda root, *_: root.get('ml_score') or 0
+ ),
+ 'uriRep': GraphQLField(
+ type=GraphQLString,
+ description='URI reputation metadata',
+ resolver=lambda root, *_: root.get('uri_rep')
+ ),
+ 'responseCodeLabel': GraphQLField(
+ type=GraphQLString,
+ description='HTTP response code name',
+ resolver=lambda root, *_: root.get('respcode_name')
+ ),
+ 'networkContext': GraphQLField(
+ type=GraphQLString,
+ description='@deprecated',
+ resolver=lambda root, *_: root.get('network_context')
+ )
+ }
+)
+
+EdgeDetailsType = GraphQLObjectType(
+ name='ProxyEdgeDetailsType',
+ fields={
+ 'datetime': GraphQLField(
+ type=GraphQLString,
+ description='Start time of the request',
+ resolver=lambda root, *_: '{} {}'.format(root.get('tdate') or '1970-01-01', root.get('time') or '00:00:00')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('clientip')
+ ),
+ 'host': GraphQLField(
+ type=GraphQLString,
+ description='Host name from the client request URL',
+ resolver=lambda root, *_: root.get('host')
+ ),
+ 'webCategory': GraphQLField(
+ type=GraphQLString,
+ description='Web content categories',
+ resolver=lambda root, *_: root.get('webcat')
+ ),
+ 'responseCode': GraphQLField(
+ type=GraphQLInt,
+ description='HTTP response code',
+ resolver=lambda root, *_: root.get('respcode') or 0
+ ),
+ 'responseCodeLabel': GraphQLField(
+ type=GraphQLString,
+ description='HTTP response code name',
+ resolver=lambda root, *_: root.get('respcode_name')
+ ),
+ 'requestMethod': GraphQLField(
+ type=GraphQLString,
+ description='HTTP request method',
+ resolver=lambda root, *_: root.get('reqmethod')
+ ),
+ 'userAgent': GraphQLField(
+ type=GraphQLString,
+ description='Client\'s user agent',
+ resolver=lambda root, *_: root.get('useragent')
+ ),
+ 'responseContentType': GraphQLField(
+ type=GraphQLString,
+ description='HTTP response content type (MIME)',
+ resolver=lambda root, *_: root.get('resconttype')
+ ),
+ 'referer': GraphQLField(
+ type=GraphQLString,
+ description='The address of the webpage that linked to the resource being requested',
+ resolver=lambda root, *_: root.get('referer')
+ ),
+ 'uriPort': GraphQLField(
+ type=GraphQLInt,
+ description='URI port',
+ resolver=lambda root, *_: root.get('uriport')
+ ),
+ 'serverIp': GraphQLField(
+ type=SpotIpType,
+ description='Server/Proxy IP',
+ resolver=lambda root, *_: root.get('serverip')
+ ),
+ 'serverToClientBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Number of bytes sent from appliance to client',
+ resolver=lambda root, *_: root.get('scbytes')
+ ),
+ 'clientToServerBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Number of bytes sent from client to appliance',
+ resolver=lambda root, *_: root.get('csbytes')
+ ),
+ 'uri': GraphQLField(
+ type=GraphQLString,
+ description='The original URI requested',
+ resolver=lambda root, *_: root.get('fulluri')
+ )
+ }
+)
+
+ScoredRequestType = GraphQLObjectType(
+ name='ProxyScoredRequestType',
+ fields={
+ 'datetime': GraphQLField(
+ type=SpotDateType,
+ description='Date and time of user score',
+ resolver=lambda root, *_: root.get('tdate') or '1970-01-01'
+ ),
+ 'uri': GraphQLField(
+ type=SpotIpType,
+ description='Requested URI',
+ resolver=lambda root, *_: root.get('fulluri')
+ ),
+ 'score': GraphQLField(
+ type=GraphQLInt,
+ description='URI risk score value. 1->High, 2->Medium, 3->Low',
+ resolver=lambda root, *_: root.get('uri_sev') or 0
+ )
+ }
+)
+
+CommentType = GraphQLObjectType(
+ name='ProxyCommentType',
+ fields={
+ 'uri': GraphQLField(
+ type=GraphQLString,
+ description='High risk URI',
+ resolver=lambda root, *_: root.get('p_threat')
+ ),
+ 'title': GraphQLField(
+ type=GraphQLString,
+ description='Threat title',
+ resolver=lambda root, *_: root.get('title')
+ ),
+ 'text': GraphQLField(
+ type=GraphQLString,
+ description='Threat description',
+ resolver=lambda root, *_: root.get('text')
+ )
+ }
+)
+
+ThreatsInformationType = GraphQLObjectType(
+ name='ProxyThreatsType',
+ fields={
+ 'list': GraphQLField(
+ type=GraphQLList(ScoredRequestType),
+ description='List of URIs that have been scored',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of scored URI. Defaults to today'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.get_scored_requests(date=args.get('date', date.today()))
+ ),
+ 'comments': GraphQLField(
+ type=GraphQLList(CommentType),
+ description='A list of comments about threats',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of high risk comments. Defaults to today'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.story_board(date=args.get('date', date.today()))
+ )
+ }
+)
+
+ThreatDetailsType = GraphQLObjectType(
+ name='ProxyThreatDetailsType',
+ fields={
+ 'datetime': GraphQLField(
+ type=SpotDatetimeType,
+ description='Start time of the request',
+ resolver=lambda root, *_: '{} {}'.format(root.get('p_date') or '1970-01-01', root.get('p_time') or '00:00:00')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('clientip')
+ ),
+ 'username': GraphQLField(
+ type=GraphQLString,
+ description='Username used for authetication',
+ resolver=lambda root, *_: root.get('username')
+ ),
+ 'duration': GraphQLField(
+ type=GraphQLInt,
+ description='Connection duration',
+ resolver=lambda root, *_: root.get('duration')
+ ),
+ 'uri': GraphQLField(
+ type=GraphQLString,
+ description='The original URI requested',
+ resolver=lambda root, *_: root.get('fulluri')
+ ),
+ 'webCategory': GraphQLField(
+ type=GraphQLString,
+ description='Web content categories',
+ resolver=lambda root, *_: root.get('webcat')
+ ),
+ 'responseCode': GraphQLField(
+ type=GraphQLInt,
+ description='HTTP response code',
+ resolver=lambda root, *_: root.get('respcode')
+ ),
+ 'requestMethod': GraphQLField(
+ type=GraphQLString,
+ description='HTTP request method',
+ resolver=lambda root, *_: root.get('reqmethod')
+ ),
+ 'userAgent': GraphQLField(
+ type=GraphQLString,
+ description='Client\'s user agent',
+ resolver=lambda root, *_: root.get('useragent')
+ ),
+ 'responseContentType': GraphQLField(
+ type=GraphQLString,
+ description='HTTP response content type (MIME)',
+ resolver=lambda root, *_: root.get('resconttype')
+ ),
+ 'referer': GraphQLField(
+ type=GraphQLString,
+ description='The address of the webpage that linked to the resource being requested',
+ resolver=lambda root, *_: root.get('referer')
+ ),
+ 'uriPort': GraphQLField(
+ type=GraphQLInt,
+ description='URI port',
+ resolver=lambda root, *_: root.get('uriport')
+ ),
+ 'serverIp': GraphQLField(
+ type=SpotIpType,
+ description='The address of the webpage that linked to the resource being requested',
+ resolver=lambda root, *_: root.get('serverip')
+ ),
+ 'serverToClientBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Number of bytes sent from appliance to client',
+ resolver=lambda root, *_: root.get('scbytes')
+ ),
+ 'clientToServerBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Number of bytes sent from client to appliance',
+ resolver=lambda root, *_: root.get('csbytes')
+ )
+ }
+)
+
+IncidentProgressionRequestType = GraphQLObjectType(
+ name='ProxyIncidentProgressionRequestType',
+ fields={
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP',
+ resolver=lambda root, *_: root.get('clientip')
+ ),
+ 'referer': GraphQLField(
+ type=GraphQLString,
+ description='The address of the webpage that linked to the resource being requested',
+ resolver=lambda root, *_: root.get('referer')
+ ),
+ 'requestMethod': GraphQLField(
+ type=GraphQLString,
+ description='HTTP Request Method',
+ resolver=lambda root, *_: root.get('reqmethod')
+ ),
+ 'responseContentType': GraphQLField(
+ type=GraphQLString,
+ description='HTTP response content type (MIME)',
+ resolver=lambda root, *_: root.get('resconttype')
+ )
+ }
+)
+
+IncidentProgressionType = GraphQLObjectType(
+ name='ProxyIncidentProgressionType',
+ fields={
+ 'uri': GraphQLField(
+ type=GraphQLString,
+ description='Threat URI',
+ resolver=lambda root, *_: root.get('fulluri')
+ ),
+ 'refererFor': GraphQLField(
+ type=GraphQLList(GraphQLString),
+ description='A list of URI who whose referer is the threat\'s URI',
+ resolver=lambda root, *_: root.get('referer_for')
+ ),
+ 'requests': GraphQLField(
+ type=GraphQLList(IncidentProgressionRequestType),
+ description='A list of requests made to Threat\'s URI',
+ resolver=lambda root, *_: root.get('requests')
+ )
+ }
+)
+
+TimelineType = GraphQLObjectType(
+ name='ProxyTimelineType',
+ fields={
+ 'startDatetime': GraphQLField(
+ type=SpotDatetimeType,
+ description='Connection\'s start time',
+ resolver=lambda root, *_: root.get('tstart') or '1970-01-01 00:00:00'
+ ),
+ 'endDatetime': GraphQLField(
+ type=SpotDatetimeType,
+ description='Connection\'s end time',
+ resolver=lambda root, *_: root.get('tend') or '1970-01-01 00:00:00'
+ ),
+ 'duration': GraphQLField(
+ type=GraphQLInt,
+ description='Connection duration',
+ resolver=lambda root, *_: root.get('duration')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('clientip')
+ ),
+ 'responseCode': GraphQLField(
+ type=GraphQLInt,
+ description='HTTP response code',
+ resolver=lambda root, *_: root.get('respcode')
+ ),
+ 'responseCodeLabel': GraphQLField(
+ type=GraphQLString,
+ description='HTTP response code name',
+ resolver=lambda root, *_: root.get('respcode_name')
+ )
+ }
+)
+
+ThreatInformationType = GraphQLObjectType(
+ name='ProxyThreatInformation',
+ fields={
+ 'details': GraphQLField(
+ type=GraphQLList(ThreatDetailsType),
+ description='Detailed information about a high risk threat',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for detailed information. Defaults to today'
+ ),
+ 'uri': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat\'s URI'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.expanded_search(date=args.get('date', date.today()), uri=args.get('uri'))
+ ),
+ 'incidentProgression': GraphQLField(
+ type=IncidentProgressionType,
+ description='Details the type of connections that conform the activity related to the threat',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for incident progression information. Defaults to today'
+ ),
+ 'uri': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat URI'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.incident_progression(date=args.get('date', date.today()), uri=args.get('uri'))
+ ),
+ 'timeline': GraphQLField(
+ type=GraphQLList(TimelineType),
+ description='Lists \'clusters\' of inbound connections to the IP, grouped by time; showing an overall idea of the times during the day with the most activity',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for time line information. Defaults to today'
+ ),
+ 'uri': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat URI'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.time_line(date=args.get('date', date.today()), uri=args.get('uri'))
+ )
+ }
+)
+
+QueryType = GraphQLObjectType(
+ name='ProxyQueryType',
+ fields={
+ 'suspicious': GraphQLField(
+ type=GraphQLList(SuspiciousType),
+ description='Proxy suspicious requests',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of suspicious requests. Defaults to today'
+ ),
+ 'uri': GraphQLArgument(
+ type=GraphQLString,
+ description='URI of interest'
+ ),
+ 'clientIp': GraphQLArgument(
+ type=SpotIpType,
+ description='Client\'s ip'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.suspicious_requests(date=args.get('date', date.today()), uri=args.get('uri'), ip=args.get('clientIp'))
+ ),
+ 'edgeDetails': GraphQLField(
+ type=GraphQLList(EdgeDetailsType),
+ description='HTTP requests to a particular URI',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of requests made by client. Defaults to today'
+ ),
+ 'uri': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLString),
+ description='URI of interest'
+ ),
+ 'clientIp': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Client\'s IP'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.details(date=args.get('date', date.today()), uri=args.get('uri'), ip=args.get('clientIp'))
+ ),
+ 'threats': GraphQLField(
+ type=ThreatsInformationType,
+ description='Advanced inforamtion about threats',
+ resolver=lambda *_ : {}
+ ),
+ 'threat': GraphQLField(
+ type=ThreatInformationType,
+ description='Advanced inforamtion about a single threat',
+ resolver=lambda *_:{}
+ ),
+ 'ingestSummary': GraphQLField(
+ type=GraphQLList(IngestSummaryType),
+ description='Summary of ingested proxy records in range',
+ args={
+ 'startDate': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='Start date'
+ ),
+ 'endDate': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='End date'
+ )
+ },
+ resolver=lambda root, args, *_: Proxy.ingest_summary(start_date=args.get('startDate'), end_date=args.get('endDate'))
+ )
+ }
+)
+
+TYPES = []
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/schema.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/schema.py b/spot-oa/api/graphql/schema.py
new file mode 100644
index 0000000..324128d
--- /dev/null
+++ b/spot-oa/api/graphql/schema.py
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from graphql import (
+ GraphQLSchema,
+ GraphQLObjectType,
+ GraphQLField
+)
+
+from flow import QueryType as NetflowQueryType, MutationType as NetflowMutationType, TYPES as NetflowTypes
+from dns import QueryType as DnsQueryType, MutationType as DnsMutationType, TYPES as DnsTypes
+from proxy import QueryType as ProxyQueryType, MutationType as ProxyMutationType, TYPES as ProxyTypes
+
+SpotSchema = GraphQLSchema(
+ query=GraphQLObjectType(
+ name='SpotQueryType',
+ fields={
+ 'flow': GraphQLField(
+ type=NetflowQueryType,
+ description='Flow is a network protocol that collects IP traffic information and monitors network traffic',
+ resolver=lambda *_: {}
+ ),
+ 'dns': GraphQLField(
+ type=DnsQueryType,
+ description='Domain Name System (DNS) Log Records contains the requests in between clients and DNS servers',
+ resolver=lambda *_: {}
+ ),
+ 'proxy': GraphQLField(
+ type=ProxyQueryType,
+ description='Proxy Logs contains the requests in between clients and Proxy servers',
+ resolver=lambda *_: {}
+ )
+ }
+ ),
+ mutation=GraphQLObjectType(
+ name='SpotMutationType',
+ fields={
+ 'flow': GraphQLField(
+ type=NetflowMutationType,
+ description='Flow related mutation operations',
+ resolver=lambda *_: {}
+ ),
+ 'dns': GraphQLField(
+ type=DnsMutationType,
+ description='DNS related mutation operations',
+ resolver=lambda *_: {}
+ ),
+ 'proxy': GraphQLField(
+ type=ProxyMutationType,
+ description='Proxy related mutation operations',
+ resolver=lambda *_: {}
+ )
+ }
+ ),
+ types=NetflowTypes + DnsTypes + ProxyTypes
+)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/webapp.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/webapp.py b/spot-oa/api/graphql/webapp.py
new file mode 100644
index 0000000..5a4bd23
--- /dev/null
+++ b/spot-oa/api/graphql/webapp.py
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+if __name__=='__main__':
+ import sys
+ from os import path
+
+ sys.path.append(path.dirname(path.dirname(path.dirname(__file__))))
+
+from flask import Flask, Blueprint
+from flask_graphql import GraphQLView
+import os
+
+from schema import SpotSchema
+
+app = Flask(__name__)
+
+blueprint = Blueprint('graphql_api', __name__)
+blueprint.add_url_rule('/graphql', strict_slashes=False, view_func=GraphQLView.as_view('graphql', schema=SpotSchema, graphiql=os.environ.get('SPOT_DEV')=='1'))
+
+app.register_blueprint(blueprint)
+
+if __name__=='__main__':
+ port = int(sys.argv[1]) if len(sys.argv)>1 else 8889
+
+ app.run(host='0.0.0.0', port=port)
+
+def load_jupyter_server_extension(nb_app):
+ import tornado.web
+ import tornado.wsgi
+
+ wsgi_app = tornado.wsgi.WSGIContainer(app)
+ nb_app.web_app.add_handlers(r'.*', [
+ (r'/graphql.*', tornado.web.FallbackHandler, dict(fallback=wsgi_app))
+ ])
+
+ nb_app.log.info('Apache Spot server extension loaded')
+ if os.environ.get('SPOT_DEV')=='1':
+ nb_app.log.warn('Apache Spot server running in dev mode (environment var SPOT_DEV=1)')
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/README.md b/spot-oa/api/resources/README.md
new file mode 100644
index 0000000..ce8b311
--- /dev/null
+++ b/spot-oa/api/resources/README.md
@@ -0,0 +1,49 @@
+
+
+# API Resources
+
+
+API Resources are the backend methods used by GraphQL to perform CRUD operations to Apache Spot (incubating) like score connections, perform threat investigation, generate a storyboard, etc.
+
+**Classes:**
+
+* resources/flow.py
+* resources/dns
+* resources/proxy
+* resources/configurator
+* resources/hdfs_client
+* resources/impala_engine.py
+
+
+
+## **Configuration Required (spot.conf):**
+API Resources use [WebHDFS REST API] (https://hadoop.apache.org/docs/r1.0.4/webhdfs.html), based on that spot.conf has been updated to include new KEYS (o variables o entries)
+
+**_Keys in HDFS section:_**
+
+* **NAME_NODE:** this key is required to setup the name node (full DNS domain or IP) to get connected to WebHDFS REST API.
+* **WEB_PORT:** Web port for WebHDFS REST API (default=50070)
+
+**_Keys in Impala section:_**
+
+* **IMPALA_DEM:** This key was part of previous release. Now that spot uses an API to connect, you need to either put the impala daemon full DNS or Server IP Address.
+* **IMPALA_PORT:** Port on which HiveServer2 client requests are served by Impala Daemons.
+
+## **Prerequisites:**
+
+#### Python:
+* setuptools
+* thrift==0.9.3
+* impyla
+* hdfs
+
+**NOTE:** all these requirements are already part of requirements.txt file, you don't need to install the python prerequisites manually. For more information go to [install python requirements](../../README.md)
+
+#### Hadoop:
+
+* Impala.
+* WebHDFS REST API.
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/__init__.py b/spot-oa/api/resources/__init__.py
new file mode 100755
index 0000000..ecb1860
--- /dev/null
+++ b/spot-oa/api/resources/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/configurator.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/configurator.py b/spot-oa/api/resources/configurator.py
new file mode 100644
index 0000000..5bda045
--- /dev/null
+++ b/spot-oa/api/resources/configurator.py
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import ConfigParser
+import os
+
+def configuration():
+
+ conf_file = "/etc/spot.conf"
+ config = ConfigParser.ConfigParser()
+ config.readfp(SecHead(open(conf_file)))
+ return config
+
+def db():
+ conf = configuration()
+ return conf.get('conf', 'DBNAME').replace("'","").replace('"','')
+
+def impala():
+ conf = configuration()
+ return conf.get('conf', 'IMPALA_DEM'),conf.get('conf', 'IMPALA_PORT')
+
+def hdfs():
+ conf = configuration()
+ name_node = conf.get('conf',"NAME_NODE")
+ web_port = conf.get('conf',"WEB_PORT")
+ hdfs_user = conf.get('conf',"HUSER")
+ hdfs_user = hdfs_user.split("/")[-1].replace("'","").replace('"','')
+ return name_node,web_port,hdfs_user
+
+def spot():
+ conf = configuration()
+ return conf.get('conf',"HUSER").replace("'","").replace('"','')
+
+class SecHead(object):
+ def __init__(self, fp):
+ self.fp = fp
+ self.sechead = '[conf]\n'
+
+ def readline(self):
+ if self.sechead:
+ try:
+ return self.sechead
+ finally:
+ self.sechead = None
+ else:
+ return self.fp.readline()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/dns.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/dns.py b/spot-oa/api/resources/dns.py
new file mode 100644
index 0000000..a9a45d6
--- /dev/null
+++ b/spot-oa/api/resources/dns.py
@@ -0,0 +1,395 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import api.resources.impala_engine as ImpalaEngine
+import api.resources.hdfs_client as HDFSClient
+import api.resources.configurator as Configuration
+from hdfs.util import HdfsError
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the suspicious dns connections in one day.
+--------------------------------------------------------------------------
+"""
+def suspicious_queries(date, ip=None, query=None,limit=250):
+
+ db = Configuration.db()
+ sq_query = ("""
+ SELECT STRAIGHT_JOIN
+ ds.unix_tstamp,frame_len,ds.ip_dst,ds.dns_qry_name,
+ dns_qry_class,dns_qry_type,dns_qry_rcode,ml_score,tld,
+ query_rep,hh,dns_qry_class_name,dns_qry_type_name,
+ dns_qry_rcode_name,network_context
+ FROM {0}.dns_scores ds
+ LEFT JOIN {0}.dns_threat_investigation dt
+ ON (ds.dns_qry_name = dt.dns_qry_name)
+ WHERE
+ ds.y={1} AND ds.m={2} AND ds.d={3}
+ AND (dt.dns_qry_name is NULL)
+ """).format(db,date.year,date.month,date.day)
+
+ sq_filter = ""
+ sq_filter += " AND ds.ip_dst = '{0}'".format(ip) if ip else ""
+ sq_filter += " AND ds.dns_qry_name LIKE '%{0}%'".format(query) if query else ""
+ sq_filter += " ORDER BY ds.ml_score limit {0}".format(limit)
+
+ sq_query = sq_query + sq_filter
+ return ImpalaEngine.execute_query_as_list(sq_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connectios related to a query name in one hour
+--------------------------------------------------------------------------
+"""
+def details(frame_time, query):
+
+ db = Configuration.db()
+ details_query = ("""
+ SELECT
+ unix_tstamp,frame_len,ip_dst,ip_src,dns_qry_name,dns_qry_class,
+ dns_qry_type,dns_qry_rcode,dns_a,dns_qry_type_name,
+ dns_qry_rcode_name,dns_qry_class_name
+ FROM
+ {0}.dns_edge
+ WHERE
+ y={1} AND m={2} AND d={3} AND hh={4} AND dns_qry_name = '{5}'
+ """).format(db,frame_time.year,frame_time.month,frame_time.day,\
+ frame_time.hour,query)
+
+ return ImpalaEngine.execute_query_as_list(details_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connections with a single client.
+--------------------------------------------------------------------------
+"""
+def client_details(date, ip):
+
+ db = Configuration.db()
+ client_query =("""
+ SELECT
+ ip_dst,dns_a,dns_qry_name,ip_dst
+ FROM
+ {0}.dns_dendro
+ WHERE
+ y={1} AND m={2} AND d={3}
+ AND ip_dst='{4}'
+ """).format(db,date.year,date.month,date.day,ip)
+
+ return ImpalaEngine.execute_query_as_list(client_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the detected suspicious connections in one day.
+--------------------------------------------------------------------------
+"""
+def incident_progression(date, query,ip):
+
+ if not ip and not query:
+ return None
+
+ db = Configuration.db()
+ return_value = "dns_qry_name" if ip else "ip_dst"
+ dns_threat_query = ("""
+ SELECT
+ anchor,total,{0}
+ FROM
+ {1}.dns_threat_dendro
+ WHERE
+ y={2} AND m={3} AND d={4}
+ AND anchor = '{5}'
+ """).format(return_value,db,date.year,date.month,date.day,\
+ query if query else ip)
+
+ return ImpalaEngine.execute_query_as_list(dns_threat_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all queries saved in threat investigation.
+--------------------------------------------------------------------------
+"""
+def comments(date):
+
+ db = Configuration.db()
+ comments_query = ("""
+ SELECT
+ ip_threat,dns_threat,title,text
+ FROM
+ {0}.dns_storyboard
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ results = ImpalaEngine.execute_query_as_list(comments_query)
+ for row in results:
+ row["text"] = row["text"].replace("\n","\\n")
+ return results
+
+"""
+--------------------------------------------------------------------------
+Score connections.
+--------------------------------------------------------------------------
+"""
+def score_connection(date,ip="", dns="", ip_sev=0, dns_sev=0):
+
+ if (not ip and not ip_sev) and (not dns and not dns_sev):
+ return False
+
+ db = Configuration.db()
+ sq_query = ("""
+ SELECT
+ frame_time,unix_tstamp,frame_len,ip_dst,dns_qry_name,dns_qry_class,
+ dns_qry_type,dns_qry_rcode,ml_score,tld,query_rep,
+ hh,dns_qry_class_name,dns_qry_type_name,dns_qry_rcode_name,
+ network_context
+ FROM
+ {0}.dns_scores
+ WHERE
+ y={1} and m={2} and d={3}
+ AND (
+ """).format(db,date.year,date.month,date.day)
+
+ connections_filter = ""
+ connections_filter += "ip_dst = '{0}' ".format(ip) if ip else ""
+ connections_filter += " OR " if ip and dns else ""
+ connections_filter += "dns_qry_name = '{0}' ".format(dns) if dns else ""
+ connections_filter += ")"
+ connections = ImpalaEngine.execute_query(sq_query + connections_filter)
+
+ # add score to connections
+
+ insert_command = ("""INSERT INTO {0}.dns_threat_investigation
+ PARTITION (y={1},m={2},d={3})
+ VALUES (""") \
+ .format(db,date.year,date.month,date.day)
+
+ fb_data = []
+ first = True
+ num_rows = 0
+ for row in connections:
+ # insert into dns_threat_investigation.
+ threat_data = (row[1],row[3],row[4],ip_sev if ip == row[3] else 0,\
+ dns_sev if dns == row[4] else 0)
+
+ fb_data.append([row[0],row[2],row[3],row[4],row[5],row[6],row[7],\
+ row[8],row[9],row[10],row[11],ip_sev,dns_sev,row[12],row[13],row[14],\
+ row[15],row[1]])
+
+ insert_command += "{0}{1}".format("," if not first else "", threat_data)
+ first = False
+ num_rows += 1
+
+ insert_command += ")"
+ if num_rows > 0: ImpalaEngine.execute_query(insert_command)
+
+ # create feedback file.
+ app_path = Configuration.spot()
+ feedback_path = "{0}/dns/scored_results/{1}{2}{3}/feedback"\
+ .format(app_path,date.year,str(date.month).zfill(2),str(date.day).zfill(2))
+ ap_file = True
+
+ if len(HDFSClient.list_dir(feedback_path)) == 0:
+ fb_data.insert(0,["frame_time","frame_len","ip_dst","dns_qry_name",\
+ "dns_qry_class","dns_qry_type","dns_qry_rcode","score","tld","query_rep",\
+ "hh","ip_sev","dns_sev","dns_qry_class_name","dns_qry_type_name",\
+ "dns_qry_rcode_name","network_context","unix_tstamp"])
+ ap_file = False
+
+ HDFSClient.put_file_csv(fb_data,feedback_path,"ml_feedback.csv",append_file=ap_file)
+ return True
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the scored connections in one day.
+--------------------------------------------------------------------------
+"""
+def get_scored_connections(date):
+
+ db = Configuration.db()
+ sc_query = ("""
+ SELECT
+ unix_tstamp,ip_dst,dns_qry_name,ip_sev,dns_sev
+ FROM
+ {0}.dns_threat_investigation
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ return ImpalaEngine.execute_query_as_list(sc_query)
+
+"""
+--------------------------------------------------------------------------
+Get expanded search from raw data table.
+--------------------------------------------------------------------------
+"""
+def expanded_search(date,query=None,ip=None,limit=20):
+
+ if not ip and not query:
+ return False
+
+ db = Configuration.db()
+ if ip:
+ count = "dns_qry_name"
+ filter_param = "ip_dst"
+ filter_value = ip
+ else:
+ count = "ip_dst"
+ filter_param = "dns_qry_name"
+ filter_value = query
+
+ expanded_query = ("""
+ SELECT
+ COUNT({0}) as total,dns_qry_name,ip_dst
+ FROM
+ {1}.dns
+ WHERE y={2} AND m={3} AND d={4}
+ AND {5} = '{6}'
+ GROUP BY {0},{5}
+ ORDER BY total DESC
+ LIMIT {7}
+ """).format(count,db,date.year,date.month,date.day,\
+ filter_param,filter_value,limit if limit else 20)
+
+ return ImpalaEngine.execute_query_as_list(expanded_query)
+
+"""
+--------------------------------------------------------------------------
+Create StoryBoard.
+--------------------------------------------------------------------------
+"""
+def create_storyboard(expanded_search,date,ip,query,title,text):
+
+ if not ip and not query:
+ return False
+
+ anchor = ip if ip else query
+ create_dendro(expanded_search,date,anchor)
+ save_comments(anchor,ip,query,title,text,date)
+
+"""
+--------------------------------------------------------------------------
+Create dendrogram for StoryBoard.
+--------------------------------------------------------------------------
+"""
+def create_dendro(expanded_search,date,anchor):
+
+ db = Configuration.db()
+ for row in expanded_search:
+ dendro_query = ("""
+ INSERT INTO {0}.dns_threat_dendro PARTITION (y={1}, m={2},d={3})
+ VALUES ( '{4}',{5},'{6}','{7}')
+ """)\
+ .format(db,date.year,date.month,date.day,anchor,\
+ row["total"],row["dnsQuery"],row["clientIp"])
+
+ ImpalaEngine.execute_query(dendro_query)
+
+"""
+--------------------------------------------------------------------------
+Create save comments for StoryBoard.
+--------------------------------------------------------------------------
+"""
+def save_comments(anchor,ip,query,title,text,date):
+
+ db = Configuration.db()
+ sb_query = ("""
+ SELECT
+ ip_threat,dns_threat,title,text
+ FROM
+ {0}.dns_storyboard
+ WHERE
+ y = {1} AND m= {2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+ sb_data = ImpalaEngine.execute_query_as_list(sb_query)
+
+ # find value if already exists.
+ saved = False
+ for item in sb_data:
+ if item["ip_threat"] == anchor or item["dns_threat"]== anchor:
+ item["title"] = title
+ item["text"] = text
+ saved = True
+
+ if not saved:
+ sb_data.append({'text': text, 'ip_threat': str(ip), 'title': title,'dns_threat':query})
+
+
+ #remove old file.
+ app_path = Configuration.spot()
+ old_file = "{0}/dns/hive/oa/storyboard/y={1}/m={2}/d={3}/"\
+ .format(app_path,date.year,date.month,date.day)
+
+ HDFSClient.delete_folder(old_file,"impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+
+ for item in sb_data:
+ insert_query = ("""
+ INSERT INTO {0}.dns_storyboard PARTITION(y={1} , m={2} ,d={3})
+ VALUES ( '{4}', '{5}', '{6}','{7}')
+ """)\
+ .format(db,date.year,date.month,date.day,\
+ item["ip_threat"],item["dns_threat"],item["title"],item["text"])
+ ImpalaEngine.execute_query(insert_query)
+
+ return True
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with all the data ingested during the timeframe
+provided.
+--------------------------------------------------------------------------
+"""
+def ingest_summary(start_date,end_date):
+
+ db = Configuration.db()
+ is_query = ("""
+ SELECT
+ tdate,total
+ FROM {0}.dns_ingest_summary
+ WHERE
+ ( y >= {1} and y <= {2}) AND
+ ( m >= {3} and m <= {4}) AND
+ ( d >= {5} and d <= {6})
+ """)\
+ .format(db,start_date.year,end_date.year,start_date.month,end_date.month, start_date.day, end_date.day)
+
+ return ImpalaEngine.execute_query_as_list(is_query)
+
+"""
+--------------------------------------------------------------------------
+Reset scored connections.
+--------------------------------------------------------------------------
+"""
+def reset_scored_connections(date):
+
+ dns_storyboard = "dns/hive/oa/storyboard"
+ dns_threat_investigation = "dns_threat_dendro/hive/oa/threat_dendro"
+ dns_timeline = "dns/hive/oa/threat_investigation"
+ app_path = Configuration.spot()
+
+ try:
+ # remove parquet files manually to allow the comments update.
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,dns_storyboard,date.year,date.month,date.day) , "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,dns_threat_investigation,date.year,date.month,date.day), "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,dns_timeline,date.year,date.month,date.day), "impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+ return True
+
+ except HdfsError:
+ return False
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/flow.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/flow.py b/spot-oa/api/resources/flow.py
new file mode 100755
index 0000000..ab5105f
--- /dev/null
+++ b/spot-oa/api/resources/flow.py
@@ -0,0 +1,958 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import api.resources.impala_engine as ImpalaEngine
+import api.resources.hdfs_client as HDFSClient
+import api.resources.configurator as Configuration
+import os,csv
+import linecache,bisect
+import struct, socket
+from hdfs.util import HdfsError
+import json
+import numpy as np
+
+"""
+--------------------------------------------------------------------------
+Return a list (dict) of all the suspicious connections that happened
+during the specified timeframe.
+--------------------------------------------------------------------------
+"""
+def suspicious_connections(date,ip=None,limit=250):
+
+ db = Configuration.db()
+ sc_query = ("""
+ SELECT STRAIGHT_JOIN
+ fs.tstart,fs.srcip,fs.dstip,fs.sport,fs.dport,proto,
+ ipkt,ibyt,opkt,obyt,ml_score,rank,srcip_internal,
+ dstip_internal,src_geoloc,dst_geoloc,src_domain,
+ dst_domain,src_rep,dst_rep
+ FROM {0}.flow_scores fs
+ LEFT JOIN {0}.flow_threat_investigation ft
+ ON (( fs.srcip = ft.srcip) OR ( fs.dstip = ft.dstip))
+ WHERE fs.y={1} AND fs.m={2} and fs.d={3}
+ AND ( ft.srcip is NULL AND ft.dstip is NULL )
+ """).format(db,date.year,date.month,date.day)
+
+ sc_filter = ""
+ if ip:
+ sc_filter = " AND ( fs.srcip='{0}' OR fs.dstip='{0}')".format(ip)
+
+ sc_filter += " ORDER BY rank limit {0}".format(limit)
+ sc_query = sc_query + sc_filter
+ return ImpalaEngine.execute_query_as_list(sc_query)
+
+"""
+--------------------------------------------------------------------------
+Retuarn a list(dict) of all the connections that happened
+between 2 IPs in one minute.
+--------------------------------------------------------------------------
+"""
+def details(src_ip,dst_ip,date):
+
+ db = Configuration.db()
+ details_query = ("""
+ SELECT
+ tstart,srcip,dstip,sport,dport,proto,flags,
+ tos,ibyt,ipkt,input,output,rip,obyt,opkt
+ FROM {0}.flow_edge
+ WHERE
+ y={1} AND m={2} AND d={3} AND hh={4} AND mn={5}
+ AND ((srcip='{6}' AND dstip='{7}')
+ OR (srcip='{7}' AND dstip='{6}'))
+ ORDER BY tstart
+ """).format(db,date.year,date.month,date.day,date.hour, \
+ date.minute,src_ip,dst_ip)
+
+ return ImpalaEngine.execute_query_as_list(details_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connections related with a single IP.
+--------------------------------------------------------------------------
+"""
+def chord_details(ip,date):
+
+ db = Configuration.db()
+ chord_query = ("""
+ SELECT
+ srcip,dstip,ibyt,ipkt
+ FROM {0}.flow_chords
+ WHERE y={1} AND m={2} AND d={3} AND ip_threat='{4}'
+ """).format(db,date.year,date.month,date.day,ip)
+
+ return ImpalaEngine.execute_query_as_list(chord_query)
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with all the data ingested during
+the timeframe provided.
+--------------------------------------------------------------------------
+"""
+def ingest_summary(start_date,end_date):
+
+ db = Configuration.db()
+ is_query = ("""
+ SELECT
+ tdate,total
+ FROM {0}.flow_ingest_summary
+ WHERE
+ ( y >= {1} AND y <= {2}) AND
+ ( m >= {3} AND m <= {4}) AND
+ ( d >= {5} AND d <= {6})
+ ORDER BY tdate
+ """).format(db,start_date.year,end_date.year, \
+ start_date.month,end_date.month, \
+ start_date.day, end_date.day)
+
+ return ImpalaEngine.execute_query_as_list(is_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of connecions that happened between 2 ip
+grouped by second.
+--------------------------------------------------------------------------
+"""
+def time_line(ip,date):
+
+ db = Configuration.db()
+ time_line_query = ("""
+ SELECT
+ ip_threat,tstart,tend,srcip,dstip,proto,
+ sport,dport,ipkt,ibyt
+ FROM {0}.flow_timeline
+ WHERE
+ y={1} AND m={2} AND d={3}
+ AND ip_threat = '{4}'
+ """).format(db,date.year,date.month,date.day,ip)
+
+ return ImpalaEngine.execute_query_as_list(time_line_query)
+
+"""
+--------------------------------------------------------------------------
+Return json file with all the geo localization information.
+--------------------------------------------------------------------------
+"""
+def sc_geo(ip,date):
+
+ app_path = Configuration.spot()
+ file_name = "globe-{0}.json".format(ip.replace(".","_"))
+ hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+ .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+ if HDFSClient.file_exists(hdfs_path,file_name):
+ return json.loads(HDFSClient.get_file("{0}/{1}" \
+ .format(hdfs_path,file_name)))
+ else:
+ return {}
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with the ip threatn information captured by the
+security expert.
+--------------------------------------------------------------------------
+"""
+def story_board(date):
+
+ db = Configuration.db()
+ sb_query= ("""
+ SELECT
+ ip_threat,title,text
+ FROM
+ {0}.flow_storyboard
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ results = ImpalaEngine.execute_query_as_list(sb_query)
+ for row in results:
+ row["text"] = row["text"].replace("\n","\\n")
+ return results
+
+"""
+--------------------------------------------------------------------------
+Return a json file with the impact analysis information.
+--------------------------------------------------------------------------
+"""
+def impact_analysis(ip,date):
+
+ app_path = Configuration.spot()
+ file_name = "stats-{0}.json".format(ip.replace(".","_"))
+ hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+ .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+ if HDFSClient.file_exists(hdfs_path,file_name):
+ return json.loads(HDFSClient.get_file("{0}/{1}" \
+ .format(hdfs_path,file_name)))
+ else:
+ return {}
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with all the inbound, outbound and twoway connections.
+--------------------------------------------------------------------------
+"""
+def incident_progression(ip,date):
+
+ app_path = Configuration.spot()
+ file_name = "threat-dendro-{0}.json".format(ip.replace(".","_"))
+
+ hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+ .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+ if HDFSClient.file_exists(hdfs_path,file_name):
+ return json.loads(HDFSClient.get_file("{0}/{1}" \
+ .format(hdfs_path,file_name)))
+ else:
+ return {}
+
+"""
+--------------------------------------------------------------------------
+Save scored connection into Threat investigation table.
+--------------------------------------------------------------------------
+"""
+def score_connection(score,date,src_ip=None,dst_ip=None,src_port=None,dst_port=None):
+
+ if not src_ip and not dst_ip and not src_port and not dst_port:
+ return False
+
+ db = Configuration.db()
+ # get connections to score
+ connections_query = ("""
+ SELECT
+ tstart,srcip,dstip,sport,dport, ibyt,ipkt
+ FROM {0}.flow_scores
+ WHERE
+ y = {1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ connections_filter = ""
+ connections_filter += " AND srcip = '{0}'".format(src_ip) if src_ip else ""
+ connections_filter += " AND dstip = '{0}'".format(dst_ip) if dst_ip else ""
+
+ connections_filter += " AND sport = {0}" \
+ .format(str(src_port)) if src_port else ""
+
+ connections_filter += " AND dport = {0}" \
+ .format(str(dst_port)) if dst_port else ""
+ connections = ImpalaEngine.execute_query(connections_query + connections_filter)
+
+
+ # add score to connections
+ insert_command = ("""
+ INSERT INTO {0}.flow_threat_investigation
+ PARTITION (y={1},m={2},d={3})
+ VALUES (""") \
+ .format(db,date.year,date.month,date.day)
+
+ fb_data = []
+ first = True
+ num_rows = 0
+ for row in connections:
+ # insert into flow_threat_investigation.
+ threat_data = (row[0],row[1],row[2],row[3],row[4],score)
+ fb_data.append([score,row[0],row[1],row[2],row[3],row[4],row[5],row[6]])
+ insert_command += "{0}{1}".format("," if not first else "", threat_data)
+ first = False
+ num_rows += 1
+
+ insert_command += ")"
+ if num_rows > 0: ImpalaEngine.execute_query(insert_command)
+
+ # create feedback file.
+ app_path = Configuration.spot()
+ feedback_path = "{0}/flow/scored_results/{1}{2}{3}/feedback" \
+ .format(app_path,date.year,str(date.month).zfill(2),str(date.day).zfill(2))
+
+ append_file = True
+ if len(HDFSClient.list_dir(feedback_path)) == 0:
+ fb_data.insert(0,["sev","tstart","sip","dip","sport","dport","ipkt","ibyt"])
+ append_file = False
+
+ HDFSClient.put_file_csv(fb_data,feedback_path,"ml_feedback.csv",\
+ append_file=append_file)
+ return True
+
+"""
+--------------------------------------------------------------------------
+Save connections details to flow_storyboard table.
+--------------------------------------------------------------------------
+"""
+def save_comment(ip,title,text,date):
+
+ #Get current table info.
+ db = Configuration.db()
+ sb_query = ("""
+ SELECT
+ ip_threat,title,text
+ FROM
+ {0}.flow_storyboard
+ WHERE
+ y = {1} AND m= {2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ sb_data = ImpalaEngine.execute_query_as_list(sb_query)
+
+ # find value if already exists.
+ saved = False
+ for item in sb_data:
+ if item["ip_threat"] == ip:
+ item["title"] = title
+ item["text"] = text
+ saved = True
+
+ if not saved:
+ sb_data.append({'text': text, 'ip_threat': str(ip), 'title': title})
+
+ #remove old file.
+ app_path = Configuration.spot()
+ old_file = "{0}/flow/hive/oa/storyboard/y={1}/m={2}/d={3}/" \
+ .format(app_path,date.year,date.month,date.day)
+
+ # remove file manually to allow the comments update.
+ HDFSClient.delete_folder(old_file,"impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+
+ for item in sb_data:
+ insert_query = ("""
+ INSERT INTO {0}.flow_storyboard PARTITION(y={1} , m={2} ,d={3})
+ VALUES ( '{4}', '{5}','{6}')
+ """) \
+ .format(db,date.year,date.month,date.day, \
+ item["ip_threat"],item["title"],item["text"])
+
+ ImpalaEngine.execute_query(insert_query)
+ return True
+
+"""
+--------------------------------------------------------------------------
+Get scored connections from threat investigation table.
+--------------------------------------------------------------------------
+"""
+def get_scored_connections(date):
+
+
+ db = Configuration.db()
+ scored_query = ("""
+ SELECT
+ tstart,srcip,dstip,srcport,dstport,score
+ FROM
+ {0}.flow_threat_investigation
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ return ImpalaEngine.execute_query_as_list(scored_query)
+
+"""
+--------------------------------------------------------------------------
+Get expanded search data from raw data table.
+--------------------------------------------------------------------------
+"""
+def expanded_search(date,ip):
+
+ db = Configuration.db()
+ expanded_query = ("""
+ SELECT
+ min(treceived) as firstseen, max(treceived) as lastseen,
+ sip as srcip, dip as dstip, sport as sport,
+ dport as dport, count(sip) as conns, max(ipkt) as maxpkts,
+ avg(ipkt) as avgpkts, max(ibyt) as maxbyts, avg(ibyt) as avgbyts
+ FROM
+ {0}.flow
+ WHERE
+ y={1} AND m={2} AND d={3}
+ AND (sip ='{4}' OR dip='{4}')
+ GROUP BY
+ sip, dip,sport,dport
+ """).format(db,date.year,date.month,date.day,ip)
+
+ return ImpalaEngine.execute_query_as_list(expanded_query)
+
+"""
+--------------------------------------------------------------------------
+Generates all the required data for StoryBoard.
+--------------------------------------------------------------------------
+"""
+def create_storyboard(expanded_search,date,ip,title,text,top_results=20):
+
+
+ cpath = "{0}/context/" \
+ .format(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+ iploc = "{0}/{1}".format(cpath,'iploc.csv')
+ nwloc = "{0}/{1}".format(cpath,'networkcontext_1.csv')
+
+ connections = get_in_out_and_twoway_conns(expanded_search,top_results)
+ inbound,outbound,twoway = add_network_context(nwloc,connections["inbound"] \
+ ,connections["outbound"],connections["twoway"])
+
+ inbound,outbound,twoway = add_geospatial_info(iploc,inbound,outbound,twoway)
+ create_impact_analysis(ip, inbound,outbound,twoway, "",date)
+ create_map_view(ip,inbound,outbound,twoway,date,iploc)
+ create_incident_progression(ip,inbound,outbound,twoway,date)
+ create_time_line(ip,inbound,outbound,twoway,date)
+ save_comment(ip,title,text,date)
+ return True
+
+"""
+--------------------------------------------------------------------------
+Calculate number of inbound only, two-way, and outbound only.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_in_out_and_twoway_conns(expanded_search,top_results=20):
+
+ inbound = {}
+ outbound = {}
+ twoway = {}
+ srcdict = {}
+ dstdict = {}
+ conns_dict= {}
+ rowct = 0
+
+ for row in expanded_search:
+ srcdict[row['srcIp']] = {
+ 'ip_int': struct.unpack("!L", socket.inet_aton(str(row['srcIp'])))[0],
+ 'dst_ip': row['dstIp'],
+ 'dst_ip_int': struct.unpack("!L", socket.inet_aton(str(row['dstIp'])))[0],
+ 'conns': int(row['connections']),
+ 'maxbytes': int(row['maxBytes'])
+ }
+ dstdict[row['dstIp']] = {
+ 'ip_int': struct.unpack("!L", socket.inet_aton(str(row['dstIp'])))[0],
+ 'src_ip': row['srcIp'],
+ 'src_ip_int': struct.unpack("!L", socket.inet_aton(str(row['srcIp'])))[0],
+ 'conns': int(row['connections']),
+ 'maxbytes': int(row['maxBytes'])
+ }
+ rowct +=1
+
+ if rowct > 0:
+ for result in srcdict:
+ if result in dstdict:
+ twoway[result] = srcdict[result]
+ else:
+ outbound[result] = srcdict[result]
+
+ for result in dstdict:
+ if result not in srcdict:
+ inbound[result] = dstdict[result]
+
+ top_inbound_b = {}
+ top_outbound_b = {}
+ top_twoway_b = {}
+
+ if len(inbound) > 0:
+ top_inbound_b = get_top_bytes(inbound,top_results)
+ top_inbound_conns = get_top_conns(inbound,top_results)
+ top_inbound_b.update(top_inbound_conns) # merge the two dictionaries
+ if len(outbound) > 0:
+ top_outbound_b = get_top_bytes(outbound,top_results)
+ top_outbound_conns = get_top_conns(outbound,top_results)
+ top_outbound_b.update(top_outbound_conns) # merge the two dictionaries
+ if len(twoway) > 0:
+ top_twoway_b = get_top_bytes(twoway,top_results)
+ top_twoway_conns = get_top_conns(twoway,top_results)
+ top_twoway_b.update(top_twoway_conns) # merge the two dictionaries
+
+
+ result = {}
+ result["inbound"] = top_inbound_b
+ result["outbound"] = top_outbound_b
+ result["twoway"] = top_twoway_b
+
+ return result
+
+"""
+--------------------------------------------------------------------------
+Create incident progression file.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_incident_progression(anchor, inbound, outbound, twoway, date):
+
+ dendro_fpath = 'threat-dendro-' + anchor.replace('.','_') + ".json"
+ obj = {
+ 'name':anchor,
+ 'children': [],
+ 'time': ""
+ }
+
+ #----- Add Inbound Connections-------#
+ if len(inbound) > 0:
+ obj["children"].append({'name': 'Inbound Only', 'children': [], 'impact': 0})
+ in_ctxs = {}
+ for ip in inbound:
+ if 'nwloc' in inbound[ip] and len(inbound[ip]['nwloc']) > 0:
+ ctx = inbound[ip]['nwloc'][2]
+ if ctx not in in_ctxs:
+ in_ctxs[ctx] = 1
+ else:
+ in_ctxs[ctx] += 1
+ for ctx in in_ctxs:
+ obj["children"][0]['children'].append({
+ 'name': ctx,
+ 'impact': in_ctxs[ctx]
+ })
+
+ #------ Add Outbound ----------------#
+ if len(outbound) > 0:
+ obj["children"].append({'name':'Outbound Only','children':[],'impact':0})
+ out_ctxs = {}
+ for ip in outbound:
+ if 'nwloc' in outbound[ip] and len(outbound[ip]['nwloc']) > 0:
+ ctx = outbound[ip]['nwloc'][2]
+ if ctx not in out_ctxs:
+ out_ctxs[ctx] = 1
+ else:
+ out_ctxs[ctx] += 1
+ for ctx in out_ctxs:
+ obj["children"][1]['children'].append({
+ 'name': ctx,
+ 'impact': out_ctxs[ctx]
+ })
+
+ #------ Add TwoWay ----------------#
+ if len(twoway) > 0:
+ obj["children"].append({'name':'two way','children': [], 'impact': 0})
+ tw_ctxs = {}
+ for ip in twoway:
+ if 'nwloc' in twoway[ip] and len(twoway[ip]['nwloc']) > 0:
+ ctx = twoway[ip]['nwloc'][2]
+ if ctx not in tw_ctxs:
+ tw_ctxs[ctx] = 1
+ else:
+ tw_ctxs[ctx] += 1
+
+ for ctx in tw_ctxs:
+ obj["children"][2]['children'].append({
+ 'name': ctx,
+ 'impact': tw_ctxs[ctx]
+ })
+
+ app_path = Configuration.spot()
+ hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+ .format(app_path,date.year,date.month,date.day,anchor.replace(".","_"))
+
+ if HDFSClient.put_file_json(obj,hdfs_path,dendro_fpath,overwrite_file=True):
+ return "Incident progression successfully created \n"
+ else:
+ return "Incident progression couldn't be created \n"
+
+"""
+--------------------------------------------------------------------------
+Create map view for StoryBoard.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_map_view(ip, inbound, outbound, twoway,date,iploc):
+
+ iplist = ''
+ globe_fpath = 'globe-' + ip.replace('.','_') + ".json"
+ if os.path.isfile(iploc):
+ iplist = np.loadtxt(iploc,dtype=np.uint32,delimiter=',',usecols={0},\
+ converters={0: lambda s: np.uint32(s.replace('"',''))})
+ else:
+ print "No iploc.csv file was found, Map View map won't be created"
+
+ response = ""
+ if iplist != '':
+
+ globe_json = {}
+ globe_json['type'] = "FeatureCollection"
+ globe_json['sourceips'] = []
+ globe_json['destips'] = []
+ for srcip in twoway:
+ try:
+ row = twoway[srcip]['geo']
+ globe_json['destips'].append({
+ 'type': 'Feature',
+ 'properties': {
+ 'location':row[8],
+ 'ip':srcip,
+ 'type':1
+ },
+ 'geometry': {
+ 'type': 'Point',
+ 'coordinates': [float(row[7]), float(row[6])]
+ }
+ })
+ except ValueError:
+ pass
+ for dstip in outbound:
+ try:
+ row = outbound[dstip]['geo']
+ dst_geo = outbound[dstip]['geo_dst']
+ globe_json['sourceips'].append({
+ 'type': 'Feature',
+ 'properties': {
+ 'location':row[8],
+ 'ip':dstip,
+ 'type':3
+ },
+ 'geometry': {
+ 'type': 'Point',
+ 'coordinates': [float(row[7]), float(row[6])]
+ }
+ })
+ globe_json['destips'].append({
+ 'type': 'Feature',
+ 'properties': {
+ 'location':row[8],
+ 'ip':outbound[dstip]['dst_ip'],
+ 'type':3
+ },
+ 'geometry': {
+ 'type': 'Point',
+ 'coordinates': [float(dst_geo[7]), float(dst_geo[6])]
+ }
+ })
+ except ValueError:
+ pass
+ for dstip in inbound:
+ try:
+ row = inbound[dstip]['geo']
+ dst_geo = inbound[dstip]['geo_src']
+ globe_json['sourceips'].append({
+ 'type': 'Feature',
+ 'properties': {
+ 'location':row[8],
+ 'ip':dstip,
+ 'type':2
+ },
+ 'geometry': {
+ 'type': 'Point',
+ 'coordinates': [float(row[7]), float(row[6])]
+ }
+ })
+ globe_json['destips'].append({
+ 'type': 'Feature',
+ 'properties': {
+ 'location':row[8],
+ 'ip':inbound[dstip]['src_ip'],
+ 'type':2
+ },
+ 'geometry': {
+ 'type': 'Point',
+ 'coordinates': [float(dst_geo[7]), float(dst_geo[6])]
+ }
+ })
+ except ValueError:
+ pass
+ json_str = json.dumps(globe_json)
+ app_path = Configuration.spot()
+ hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+ .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+ if HDFSClient.put_file_json(globe_json,hdfs_path,globe_fpath,overwrite_file=True) :
+ response = "Geolocation map successfully created \n"
+ else:
+ response = "The map can't be created without an iploc file \n"
+
+ return response
+
+"""
+--------------------------------------------------------------------------
+Create timeline for storyboard.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_time_line(anchor,inbound, outbound, twoway,date):
+
+ top_keys = []
+ if len(twoway) > 0: top_keys.extend(twoway.keys())
+ if len(outbound) > 0: top_keys.extend(outbound.keys())
+ if len(inbound) > 0: top_keys.extend(inbound.keys())
+
+
+ db = Configuration.db()
+
+ imp_query =("""
+ INSERT INTO TABLE {0}.flow_timeline PARTITION (y={4}, m={5},d={6})
+ SELECT
+ '{7}' ,min(treceived) as tstart, max(treceived) as tend,
+ sip as srcIP,dip as dstip, proto as proto, sport as sport,
+ dport AS dport, ipkt as ipkt, ibyt as ibyt
+ FROM
+ {0}.flow
+ WHERE y={4} AND m={5} AND d={6}
+ AND ((dip IN({1}) AND sip ='{2}') OR (sip IN({1}) AND dip ='{2}'))
+ GROUP BY sip, dip, proto, sport, dport, ipkt, ibyt
+ ORDER BY tstart
+ LIMIT {3}
+ """)
+
+ ips = "'" + "','".join(top_keys) + "'"
+ imp_query = imp_query.format(db,ips,anchor,1000,date.year,date.month, date.day,anchor)
+
+ if ImpalaEngine.execute_query(imp_query):
+ return "Timeline successfully created \n"
+ else:
+ return "Timeline couldn't be created \n"
+
+"""
+--------------------------------------------------------------------------
+Create Impact Analysis for StoryBoard.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_impact_analysis(anchor, inbound, outbound, twoway, threat_name,date):
+
+ stats_fpath = 'stats-' + anchor.replace('.','_') + ".json"
+
+ obj = {
+ 'name':threat_name,
+ 'children': [],
+ 'size': len(inbound) + len(outbound) + len(twoway)
+ }
+
+ #----- Add Inbound Connections-------#
+ obj["children"].append({'name': 'Inbound Only', 'children': [], 'size': len(inbound)})
+ in_ctxs = {}
+ for ip in inbound:
+ full_ctx = ''
+ if 'nwloc' in inbound[ip] and len(inbound[ip]['nwloc']) > 0:
+ full_ctx = inbound[ip]['nwloc'][2].split('.')[0]
+ ctx = get_ctx_name(full_ctx)
+ if ctx not in in_ctxs:
+ in_ctxs[ctx] = 1
+ else:
+ in_ctxs[ctx] += 1
+ for ctx in in_ctxs:
+ obj["children"][0]['children'].append({
+ 'name': ctx,
+ 'size': in_ctxs[ctx]
+ })
+
+
+ #------ Add Outbound ----------------#
+ obj["children"].append({'name':'Outbound Only','children':[],'size':len(outbound)})
+ out_ctxs = {}
+ for ip in outbound:
+ full_ctx = ''
+ if 'nwloc' in outbound[ip] and len(outbound[ip]['nwloc']) > 0:
+ full_ctx = outbound[ip]['nwloc'][2].split('.')[0]
+ ctx = get_ctx_name(full_ctx)
+ if ctx not in out_ctxs:
+ out_ctxs[ctx] = 1
+ else:
+ out_ctxs[ctx] += 1
+ for ctx in out_ctxs:
+ obj["children"][1]['children'].append({
+ 'name': ctx,
+ 'size': out_ctxs[ctx]
+ })
+
+ #------ Add Twoway ----------------#
+ obj["children"].append({'name': 'two way', 'children': [], 'size': len(twoway)})
+ tw_ctxs = {}
+ for ip in twoway:
+ full_ctx = ''
+ if 'nwloc' in twoway[ip] and len(twoway[ip]['nwloc']) > 0:
+ full_ctx = twoway[ip]['nwloc'][2].split('.')[0]
+ ctx = get_ctx_name(full_ctx)
+ if ctx not in tw_ctxs:
+ tw_ctxs[ctx] = 1
+ else:
+ tw_ctxs[ctx] += 1
+
+ for ctx in tw_ctxs:
+ obj["children"][2]['children'].append({
+ 'name': ctx,
+ 'size': tw_ctxs[ctx]
+ })
+
+ app_path = Configuration.spot()
+ hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+ .format(app_path,date.year,date.month,date.day,anchor.replace(".","_"))
+
+ data = json.dumps(obj)
+ if HDFSClient.put_file_json(obj,hdfs_path,stats_fpath,overwrite_file=True):
+ return "Stats file successfully created \n"
+ else:
+ return "Stats file couldn't be created \n"
+
+"""
+--------------------------------------------------------------------------
+Get topbytess list.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_top_bytes(conns_dict, top):
+ topbytes = sorted(conns_dict.iteritems(),key=lambda (x,y): y['maxbytes'],reverse=True)
+ topbytes = topbytes[0:top]
+ return dict(topbytes)
+
+"""
+--------------------------------------------------------------------------
+Get top connections.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_top_conns(conns_dict, top):
+ topconns = sorted(conns_dict.iteritems(), key=lambda (x,y): y['conns'], reverse=True)
+ topconns = topconns[0:top]
+ return dict(topconns)
+
+"""
+--------------------------------------------------------------------------
+Get network context - get start and end ranges.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def add_network_context(nwloc,inbound,outbound,twoway):
+ nwdict = {}
+ if os.path.isfile(nwloc) :
+ with open(nwloc, 'r') as f:
+ reader = csv.reader(f,delimiter=',')
+ reader.next()
+ #address range, description
+ for row in reader:
+
+ if '/' in row[0]:
+ #Range in subnet
+ iprange = row[0].split('/')
+ if len(iprange) < 2:
+ ipend = 0
+ else:
+ ipend = int(iprange[1])
+ nwdict[row[0]] = [struct.unpack("!L", \
+ socket.inet_aton(iprange[0]))[0],\
+ struct.unpack("!L",socket.inet_aton(iprange[0]))[0]+2**(32-ipend)-1, row[1]]
+ elif '-' in row[0]:
+ #IP Range
+ iprange = row[0].split('-')
+ nwdict[row[0]] = [struct.unpack("!L",\
+ socket.inet_aton(iprange[0].replace(" ", "")))[0],\
+ struct.unpack("!L", socket.inet_aton(iprange[1].replace(" ", "")))[0], row[1]]
+ else:
+ #Exact match
+ nwdict[row[0]] = [struct.unpack("!L",\
+ socket.inet_aton(row[0]))[0],struct.unpack("!L",\
+ socket.inet_aton(row[0]))[0], row[1]]
+
+ for srcip in outbound:
+ temp_ip = struct.unpack("!L", socket.inet_aton(srcip))[0]
+ if srcip in nwdict:
+ inbound[srcip]['nwloc'] = nwdict[srcip]
+ else:
+ matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]
+ outbound[srcip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''
+
+ for dstip in twoway:
+ temp_ip = struct.unpack("!L", socket.inet_aton(dstip))[0]
+ if dstip in nwdict:
+ twoway[dstip]['nwloc'] = nwdict[dstip]
+ else:
+ matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]
+ twoway[dstip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''
+
+ for srcip in inbound:
+ temp_ip = struct.unpack("!L", socket.inet_aton(srcip))[0]
+ if srcip in nwdict:
+ inbound[srcip]['nwloc'] = nwdict[srcip]
+ else:
+ matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]
+ inbound[srcip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''
+
+ return inbound,outbound,twoway
+
+
+"""
+--------------------------------------------------------------------------
+Add Geo spatial info
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def add_geospatial_info(iploc,inbound,outbound,twoway):
+ iplist = ''
+ if os.path.isfile(iploc):
+ iplist = np.loadtxt(iploc,dtype=np.uint32,delimiter=',',usecols={0},\
+ converters={0: lambda s: np.uint32(s.replace('"',''))})
+ else:
+ print "No iploc.csv file was found, Map View map won't be created"
+
+
+ # get geospatial info, only when iplocation file is available
+ if iplist != '':
+ for srcip in outbound:
+ reader = csv.reader([linecache.getline(\
+ iploc, bisect.bisect(iplist,outbound[srcip]['ip_int'])).replace('\n','')])
+
+ outbound[srcip]['geo'] = reader.next()
+ reader = csv.reader([linecache.getline(\
+ iploc, bisect.bisect(iplist,outbound[srcip]['dst_ip_int'])).replace('\n','')])
+ outbound[srcip]['geo_dst'] = reader.next()
+
+ for dstip in twoway:
+ reader = csv.reader([linecache.getline(\
+ iploc,bisect.bisect(iplist,twoway[dstip]['ip_int'])).replace('\n','')])
+ twoway[dstip]['geo'] = reader.next()
+
+ for srcip in inbound:
+ reader = csv.reader([linecache.getline(\
+ iploc, bisect.bisect(iplist,inbound[srcip]['ip_int'])).replace('\n','')])
+
+ inbound[srcip]['geo'] = reader.next()
+ reader = csv.reader([linecache.getline(\
+ iploc, bisect.bisect(iplist,inbound[srcip]['src_ip_int'])).replace('\n','')])
+ inbound[srcip]['geo_src'] = reader.next()
+
+ return inbound,outbound,twoway
+
+"""
+--------------------------------------------------------------------------
+Get context name.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_ctx_name(full_context):
+ ctx= 'DMZ'
+ if "VPN" in full_context:
+ ctx = "VPN"
+ elif "DMZ" in full_context:
+ ctx = "DMZ"
+ elif "Proxy" in full_context:
+ ctx = "Proxy"
+ elif "FW" in full_context:
+ ctx = "FW"
+ return ctx
+
+"""
+--------------------------------------------------------------------------
+Reset scored connections.
+--------------------------------------------------------------------------
+"""
+def reset_scored_connections(date):
+
+ flow_storyboard = "flow/hive/oa/storyboard"
+ flow_threat_investigation = "flow/hive/oa/threat_investigation"
+ flow_timeline = "flow/hive/oa/timeline"
+ app_path = Configuration.spot()
+
+ try:
+ # remove parquet files manually to allow the comments update.
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,flow_storyboard,date.year,date.month,date.day) , "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,flow_threat_investigation,date.year,date.month,date.day), "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,flow_timeline,date.year,date.month,date.day), "impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+ return True
+
+ except HdfsError:
+ return False
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/hdfs_client.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/hdfs_client.py b/spot-oa/api/resources/hdfs_client.py
new file mode 100644
index 0000000..31c5eba
--- /dev/null
+++ b/spot-oa/api/resources/hdfs_client.py
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from hdfs import InsecureClient
+from hdfs.util import HdfsError
+from json import dump
+import api.resources.configurator as Config
+
+
+def _get_client(user=None):
+ hdfs_nm,hdfs_port,hdfs_user = Config.hdfs()
+ client = InsecureClient('http://{0}:{1}'.format(hdfs_nm,hdfs_port), user= user if user else hdfs_user)
+ return client
+
+def get_file(hdfs_file):
+ client = _get_client()
+ with client.read(hdfs_file) as reader:
+ results = reader.read()
+ return results
+
+def put_file_csv(hdfs_file_content,hdfs_path,hdfs_file_name,append_file=False,overwrite_file=False):
+
+ try:
+ client = _get_client()
+ hdfs_full_name = "{0}/{1}".format(hdfs_path,hdfs_file_name)
+ with client.write(hdfs_full_name,append=append_file,overwrite=overwrite_file) as writer:
+ for item in hdfs_file_content:
+ data = ','.join(str(d) for d in item)
+ writer.write("{0}\n".format(data))
+ return True
+
+ except HdfsError:
+ return False
+
+def put_file_json(hdfs_file_content,hdfs_path,hdfs_file_name,append_file=False,overwrite_file=False):
+
+ try:
+ client = _get_client()
+ hdfs_full_name = "{0}/{1}".format(hdfs_path,hdfs_file_name)
+ with client.write(hdfs_full_name,append=append_file,overwrite=overwrite_file,encoding='utf-8') as writer:
+ dump(hdfs_file_content, writer)
+ return True
+ except HdfsError:
+ return False
+
+
+def delete_folder(hdfs_file,user=None):
+ client = _get_client(user)
+ client.delete(hdfs_file,recursive=True)
+
+def list_dir(hdfs_path):
+ try:
+ client = _get_client()
+ return client.list(hdfs_path)
+ except HdfsError:
+ return {}
+
+def file_exists(hdfs_path,file_name):
+ files = list_dir(hdfs_path)
+ if str(file_name) in files:
+ return True
+ else:
+ return False
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/impala_engine.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/impala_engine.py b/spot-oa/api/resources/impala_engine.py
new file mode 100644
index 0000000..b7d0148
--- /dev/null
+++ b/spot-oa/api/resources/impala_engine.py
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from impala.dbapi import connect
+import api.resources.configurator as Config
+
+def create_connection():
+
+ impala_host, impala_port = Config.impala()
+ db = Config.db()
+ conn = connect(host=impala_host, port=int(impala_port),database=db)
+ return conn.cursor()
+
+def execute_query(query,fetch=False):
+
+ impala_cursor = create_connection()
+ impala_cursor.execute(query)
+
+ return impala_cursor if not fetch else impala_cursor.fetchall()
+
+def execute_query_as_list(query):
+
+ query_results = execute_query(query)
+ row_result = {}
+ results = []
+
+ for row in query_results:
+ x=0
+ for header in query_results.description:
+ row_result[header[0]] = row[x]
+ x +=1
+ results.append(row_result)
+ row_result = {}
+
+ return results
+
+
[17/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/open-data-model/open-data-model.md
----------------------------------------------------------------------
diff --git a/docs/open-data-model/open-data-model.md b/docs/open-data-model/open-data-model.md
deleted file mode 100644
index 768fd52..0000000
--- a/docs/open-data-model/open-data-model.md
+++ /dev/null
@@ -1,892 +0,0 @@
-**Open Data Model (ODM)**
-=========================
-
-**Overview**
-------------
-
-This document describes a strategy for creating an open data model (ODM) for
-Apache Spot (incubating) (formerly known as “Open Network Insight (ONI)”) in
-support of cyber security analytic use cases. It also describes the use cases
-for which Apache Spot (incubating) running on the Cloudera platform is uniquely
-capable of addressing along with the data model.
-
-**Apache Spot (incubating) Open Data Model Strategy**
------------------------------------------------------
-
-The Apache Spot (incubating) Open Data Model (ODM) strategy aims to extend
-Apache Spot (incubating) capabilities to support a broader set of cyber security
-use cases than initially supported. The primary use case initially supported by
-Apache Spot (incubating) includes Network Traffic Analysis for network flows
-(Netflow, sflow, etc.), DNS and Proxy; primarily the identification of threats
-through anomalous event detection using both supervised and unsupervised machine
-learning.
-
-In order to support a broader set of use cases, Spot must be extended to collect
-and analyze other common “event-oriented” data sources analyzed for cyber
-threats, including but not limited to the following log types:
-
-- Proxy
-
-- Web server
-
-- Operating system
-
-- Firewall
-
-- Intrusion Prevention/Detection (IDS/ IPS)
-
-- Data Loss Prevention
-
-- Active Directory / Identity Management
-
-- User/Entity Behavior Analysis
-
-- Endpoint Protection/Asset Management
-
-- Network Metadata/Session and PCAP files
-
-- Network Access Control
-
-- Mail
-
-- VPN
-
-- etc..
-
-One of the biggest challenges organizations face today in combating cyber
-threats is collecting and normalizing data from the myriad of security event
-data sources (hundreds) in order to build the needed analytics. This often
-results in the analytics being dependent upon the specific technologies used by
-an organization to detect threats and prevents the needed flexibility and
-agility to keep up with these ever-increasing (and complex) threats. Technology
-lock-in is sometimes a byproduct of today’s status quo, as it’s extremely costly
-to add new technologies (or replace existing ones) because of the downstream
-analytic dependencies.
-
-To achieve the goal of extending Apache Spot (incubating) to support additional
-use cases, it is necessary to create an open data model for the most relevant
-security event and contextual data sources; Security event logs or alerts,
-Network context, User details and information that comes from the endpoints or
-any other console that are being use to manage the security / administration of
-our endpoints. The presence of an open data model, which can be applied
-“on-read” or “on-write”, in batch or stream, will allow for the separation of
-security analytics from the specific data sources on which they are built. This
-“separation of duties” will enable organizations to build analytics that are not
-dependent upon specific technologies and provide the flexibility to change
-underlying data sources and also provide segmentation of this information,
-without impacting the analytics. This will also afford security vendors the
-opportunity to build additional products on top of the Open Data Model to drive
-new revenue streams and also to design new ways to detect threats and APT.
-
-**Apache Spot (incubating) Enabled Use Cases**
-----------------------------------------------
-
-Spot on the Cloudera platform is uniquely positioned to help address the
-following cyber security use cases, which are not effectively addressed by
-legacy technologies:
-
-**- Detection of known & unknown threats leveraging machine learning and
-advanced analytic modeling**
-
-Current technologies are limited in the analytics they can apply to detect
-threats. These limitations stem from the inability to collect all the data
-sources needed to effectively identify threats (structured, unstructured, etc.)
-and inability to process the massive volumes of data needed to do so (billions
-of events per day). Legacy technologies are typically focus and limited to
-rules-based and signature detection. They are somewhat “effective” at detecting
-known threats but struggle with new threats.
-
-Spot addresses these gaps through its ability to collect any data type of any
-volume. Coupled with the various analytic frameworks that are provided
-(including machine learning), Spot enables a whole new class of analytics that
-can scale to today’s demands. The topic model used by Spot to detect anomalous
-network traffic is one example of where the Spot platform excels.
-
-**- Reduction of mean time to incident detection & resolution (MTTR)**
-
-One of the challenges organizations face today is detecting threats early enough
-to minimize adverse impacts. This stems from the limitations previously
-discussed with regards to limited analytics. It can also be attributed to the
-fact that most of the investigative queries often take hours or days to return
-results. Legacy technologies can’t offer or have a central data store for
-facilitating such investigations due to their inability to store and serve the
-massive amounts of data involved. This cripples incident investigations and
-results in MTTRs of many weeks or months, meanwhile the adverse impacts of the
-breach are magnified, thus making the threat harder to eradicate.
-
-Apache Spot (incubating) addresses these gaps by providing the capability for a
-central data store that houses ALL the data needed to facilitate an
-investigation, returning investigative query results in seconds and minutes (vs.
-hours and days). Spot can effectively reduce incident MTTR and reduce adverse
-impacts of a breach.
-
-**- Threat Hunting**
-
-It’s become necessary for organizations to “hunt” for active threats as
-traditional passive threat detection approaches are not sufficient. “Hunting”
-involves performing ad-hoc searches and queries over vast amounts of data
-representing many weeks and months’ worth of events, as well as applying ad-hoc
-/ tune algorithms to detect the needle in the haystack. Traditional systems do
-not perform well for these types of activities as the query results sometimes
-take hours and days to be retrieved. These traditional systems also lack the
-analytic flexibility to construct the necessary algorithms and logic needed.
-
-Apache Spot (incubating) addresses these gaps in the same ways it addresses
-others; by providing a central data store with the needed analytic frameworks
-that scale to the needed workloads.
-
-**Data Model**
---------------
-
-In order to provide a framework for effectively analyzing data for cyber
-threats, it is necessary to collect and analyze standard security event
-logs/alerts and contextual data regarding the entities referenced in these
-logs/alerts. The most common entities include network, user and endpoint, but
-there are others such as file.
-
-In the diagram below, the raw event tells us that user “jsmith” successfully
-logged in to an Oracle database from the IP address 10:1.1.3. Based on the raw
-event only, we don’t know if this event is a legitimate threat or not. After
-injecting user and endpoint context, the enriched event tells us this event is a
-potential threat that requires further investigation.
-
-
-
-Based on the need to collect and analyze both security events, logs or alerts
-and contextual data, support for the following types of security information are
-planned for inclusion in the Spot Open Data Model:
-
-- Security event logs/alerts This data type includes event logs from common
- data sources used to detect threats and includes network flows, operating
- system logs, IPS/IDS logs, firewall logs, proxy logs, web logs, DLP logs,
- etc.
-
-- Network context data This data type includes information about the network,
- which can be gleaned from Whois servers, asset databases and other similar
- data sources.
-
-- User context data This data type includes information from user and identity
- management systems including Active Directory, Centrify, and other identity
- and access management systems.
-
-- Endpoint context data This data includes information about endpoint systems
- (servers, workstations, routers, switches, etc.) and can be sourced from
- asset management systems, vulnerability scanners, and endpoint
- management/detection/response systems such as Webroot, Tanium, Sophos,
- Endgame, CarbonBlack, Intel Security ePO and others.
-
-- File context data **(ROADMAP ITEM)** This data includes contextual
- information about files and can be sourced from systems such as FireEye,
- Application Control , Intel Security McAfee Threat Intelligence Exchange
- (TIE).
-
-- Threat intelligence context data **(ROADMAP ITEM)** This data includes
- contextual information about URLs, domains, websites, files and others.
-
-**Naming Convention**
----------------------
-
-A naming convention is needed for the Open Data Model to represent common
-attributes across vendor products and technologies. The naming convention is
-described below.
-
-**Prefixes**
-------------
-
-| Prefix | Description |
-|----------|-----------------------------------------------------------------------------------------------------------------------------------|
-| src | Corresponds to the “source” fields within a given event (i.e. source address) |
-| dst | Corresponds to the “destination” fields within a given event (i.e. destination address) |
-| dvc | Corresponds to the “device” applicable fields within a given event (i.e. device address) and represent where the event originated |
-| fwd | Forwarded from device |
-| request | Corresponds to requested values (vs. those returned, i.e. “requested URI”) |
-| response | Corresponds to response value (vs. those requested) |
-| file | Corresponds to the “file” fields within a given event (i.e. file type) |
-| user | Corresponds to user attributes (i.e. name, id, etc.) |
-| xlate | Corresponds to translated values within a given event (i.e. src_xlate_ip for “translated source ip address” |
-| in | Ingress |
-| out | Egress |
-| new | New value |
-| orig | Original value |
-| app | Corresponds to values associated with application events |
-
-**Security Event Log/Alert Data Model**
----------------------------------------
-
-The data model for security event logs/alerts is detailed in the below. The
-attributes are categorized as follows:
-
-- Common -attributes that are common across many device types
-
-- Device -attributes that are applicable to the device that generated the
- event
-
-- File -attributes that are applicable to file objects referenced in the event
-
-- Endpoint -attributes that are applicable to the endpoints referenced in the
- event
-
-- User- attributes that are applicable to the user referenced in the event
-
-- Proxy - attributes that are applicable to proxy events
-
-- Protocol
-
-- DNS - attributes that are specific to DNS events
-
-- HTTP - attributes that are specific to HTTP events
-
-- SMTP, SSH, TLS, DHCP, IRC, SNMP and FTP
-
-Note: The model will evolve to include reserved attributes for additional device
-types that are not currently represented. The model can currently be extended to
-support ANY attribute for ANY device type by following the guidance outlined in
-the section titled [“Extensibility of Data Model”.](#extensibility)
-
-Note: Attributes denoted in **Bold**, represent those that are listed in the
-model multiple times for the purpose of demonstrating attribute coverage for a
-particular entity (endpoint, user, network, etc.) or log type (Proxy, DNS,
-etc.).
-
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-|--------------|---------------------------|-------------------|-----------------------------------------------------------------------|-------------------------------------------------------------------------------------|
-| **Common** | eventtime | long | timestamp of event (UTC) | 1472653952 |
-| | duration | int | Time duration (milliseconds) | 2345 |
-| | eventid | string | Unique identifier for event | x:2388 |
-| | org | string | Organization | “HR” or “Finance” or “CustomerA” |
-| | type | string | Type information | “Informational”, “image/gif” |
-| | nproto | string | Network protocol of event | TCP, UDP, ICMP |
-| | aproto | string | Application protocol of event | HTTP, NFS, FTP |
-| | msg | string | Message (details of action taken on object) | Some long string |
-| | mac | string | MAC address | 94:94:26:3:86:16 |
-| | severity | string | Severity of event | High, 10, 1 |
-| | raw | string | Raw text message of entire event | Complete copy of log entry |
-| | risk | Floating point | Risk score | 95.67 |
-| | code | string | Response or error code | 404 |
-| | category | string | Event category | /Application/Start |
-| | qry | string | Query (DNS query, URI query, SQL query, etc.) | Select \* from "table" |
-| | service | string | (i.e. service name, type of service) | sshd |
-| | state | string | State of object | Running, Paused, stopped |
-| | in_bytes | int | Bytes in | 1025 |
-| | out_bytes | int | Bytes out | 9344 |
-| | additional_attrs | String (JSON Map) | Custom event attributes | "building":"729","cube":"401" |
-| | dvc_time | long | UTC timestamp from device where event/alert originates or is received | 1472653952 |
-| | dvc_ip4/dvc_ip6 | long | IP address of device | Integer representaion of 10.1.1.1 |
-| | dvc_host | string | Hostname of device | Integer representaion of 10.1.1.1 |
-| | dvc_type | string | Device type that generated the log | Unix, Windows, Sonicwall |
-| | dvc_vendor | string | Vendor | Microsoft, Fireeye, Intel Security |
-| | dvc_version | string | Version | 5.4 |
-| | fwd_ip4/fwd_ip6 | long | Forwarded from device | Integer representation of 10.1.1.1 |
-| | version | string | Version | “3.2.2” |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **Network** | src_ip4/src_ip6 | bigint | Source ip address of event | Integer representation of 10.1.1.1 |
-| | src_host | string | Source FQDN of event | test.companyA.com |
-| | src_domain | string | Domain name of source address | companyA.com |
-| | src_port | int | Source port of event | 1025 |
-| | src_country_code | string | Source country code | cn |
-| | src_country_name | string | Source country name | China |
-| | src_region | string | Source region | string |
-| | src_city | string | Source city | Shenghai |
-| | src_lat | int | Source latitude | |
-| | src_long | int | Source longitude | |
-| | dst_ip4/dst_ip6 | bigint | Destination ip address of event | Integer representaion of 10.1.1.1 |
-| | dst_host | string | Destination FQDN of event | test.companyA.com |
-| | dst_domain | string | Domain name of destination address | companyA.com |
-| | dst_port | int | Destination port of event | 80 |
-| | dst_country_code | string | Source country code | cn |
-| | dst_country_name | string | Source country name | China |
-| | dst_region | string | Source region | string |
-| | dst_city | string | Source city | Shenghai |
-| | dst_lat | int | Source latitude | |
-| | dst_long | int | Source longitude | |
-| | asn | int | Autonomous system number | 33 |
-| | **in_bytes** | int | Bytes in | 987 |
-| | **out_bytes** | int | Bytes out | 1222 |
-| | direction | string | Direction | In, inbound, outbound, ingress, egress |
-| | flags | string | TCP flags | .AP.SF |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **File** | file_name | string | Filename from event | output.csv |
-| | file_path | string | File path | /root/output.csv |
-| | file_atime | bigint | Timestamp (UTC) of file access | 1472653952 |
-| | file_acls | string | File permissions | rwx-rwx-rwx |
-| | file_type | string | Type of file | “.doc” |
-| | file_size | int | Size of file in bytes | 1244 |
-| | file_desc | string | Description of file | Project Plan for Project xyz |
-| | file_hash | string | Hash of file | |
-| | file_hash_type | string | Type of hash | MD5, SHA1,SHA256 |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **Endpoint** | object | string | File/Process/Registry | File, Registry, Process |
-| | action | string | Action taken on object (open/delete/edit) | Open, Edit |
-| | **msg** | string | Message (details of action taken on object) | Some long string |
-| | app | string | Application | Microsoft Powerpoint |
-| | location | string | Location | Atlanta, GA |
-| | proc | string | Process | SSHD |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **User** | user_name | string | username from event | mhicks |
-| | email | string | Email address | test\@companyA.com |
-| | user_id | string | userid | 234456 |
-| | user_loc | string | location | Herndon, VA |
-| | user_desc | string | Description of user | |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **DNS** | dns_class | string | DNS class | 1 |
-| | dns_length | int | DNS frame length | 188 |
-| | **dns_qry** | string | Requested DNS query | test.test.com |
-| | **dns_code** | string | Response code | 0x00000001 |
-| | dns_response_qry | string | Response to DNS Query | 178.2.1.99 |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **Proxy** | **category** | string | Event category | SG-HTTP-SERVICE |
-| | browser | string | Web browser | Internet Explorer |
-| | **code** | string | Error or response code | 404 |
-| | **in_bytes** | int | Bytes in | 1025 |
-| | **out_bytes** | int | Bytes out | 1288 |
-| | referrer | string | Referrer | www.usatoday.com |
-| | **request_uri** | string | Requested URI | /wcm/assets/images/imagefileicon.gif |
-| | filter_rule | string | Applied filter or rule | Internet, Rule 6 |
-| | filter_result | string | Result of applied filter or rule | Proxied, Blocked |
-| | **qry** | string | URI query | ?func=S_senseHTML&Page=a26815a313504697a126279 |
-| | **action** | string | Action taken on object | TCP_HIT, TCP_MISS, TCP_TUNNELED |
-| | method | string | HTTP method | GET, CONNECT, POST |
-| | **type** | string | Type of request | image/gif |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **HTTP** | request_method | string | HTTP method | GET, CONNECT, POST |
-| | **request_uri** | string | Requested URI | /wcm/assets/images/imagefileicon.gif |
-| | request_body_len | int | Length of request body | 98 |
-| | request_user_name | string | username from event | mhicks |
-| | request_password | string | Password from event | abc123 |
-| | request_proxied | string | | |
-| | request_headers | MAP | HTTP request headers | request_headers[‘HOST’] request_headers[‘USER-AGENT’] request_headers[‘ACCEPT’] |
-| | response_status_code | int | HTTP response status code | 404 |
-| | response_status_msg | string | HTTP response status message | “Not found” |
-| | response_body_len | int | Length of response body | 98 |
-| | response_info_code | int | HTTP response info code | 100 |
-| | response_info_msg | string | HTTP response info message | “Some string” |
-| | response_resp_fuids | string | Response FUIDS | |
-| | response_mime_types | string | Mime types | “cgi,bat,exe” |
-| | response_headers | MAP | Response headers | response_headers[‘SERVER’] response_headers[‘SET-COOKIE’’] response_headers[‘DATE’] |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **SMTP** | trans_depth | int | Depth of email into SMTP exchange | Coming soon |
-| | headers_helo | string | Helo header | Coming soon |
-| | headers_mailfrom | string | Mailfrom header | Coming soon |
-| | headers_rcptto | string | Rcptto header | Coming soon |
-| | headers_date | string | Header date | Coming soon |
-| | headers_from | string | From header | Coming soon |
-| | headers_to | string | To header | Coming soon |
-| | headers_reply_to | string | Reply to header | Coming soon |
-| | headers_msg_id | string | Message ID | Coming soon |
-| | headers_in_reply_to | string | In reply to header | Coming soon |
-| | headers_subject | string | Subject | Coming soon |
-| | headers_x_originating_ip4 | bigint | Originating IP address | Coming soon |
-| | headers_first_received | string | First to receive message | Coming soon |
-| | headers_second_received | string | Second to receive message | Coming soon |
-| | last_reply | string | Last reply in message chain | Coming soon |
-| | path | string | Path of message | Coming soon |
-| | user_agent | string | User agent | Coming soon |
-| | tls | boolean | Indication of TLS use | Coming soon |
-| | is_webmail | boolean | Indication of webmail | Coming soon |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **FTP** | **user_name** | string | Username | Coming soon |
-| | password | string | Password | Coming soon |
-| | command | string | FTP command | Coming soon |
-| | arg | string | Argument | Coming soon |
-| | mime_type | string | Mime type | Coming soon |
-| | file_size | int | File size | Coming soon |
-| | reply_code | int | Reply code | Coming soon |
-| | reply_msg | string | Reply message | Coming soon |
-| | data_channel_passive | boolean | Passive data channel? | Coming soon |
-| | data_channel_rsp_p | string | | Coming soon |
-| | cwd | string | Current working directory | Coming soon |
-| | cmdarg_ts | float | | Coming soon |
-| | cmdarg_cmd | string | Command | Coming soon |
-| | cmdarg_arg | string | Command argument | Coming soon |
-| | cmdarg_seq | int | Sequence | Coming soon |
-| | pending_commands | string | Pending commands | Coming soon |
-| | is_passive | boolean | Passive mode enabled | Coming soon |
-| | fuid | string | Coming soon | Coming soon |
-| | last_auth_requested | string | Coming soon | Coming soon |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **SNMP** | **version** | string | Coming soon | Coming soon |
-| | community | string | Coming soon | Coming soon |
-| | get_requests | int | Coming soon | Coming soon |
-| | get_bulk_requests | int | Coming soon | Coming soon |
-| | get_responses | int | Coming soon | Coming soon |
-| | set_requests | int | Coming soon | Coming soon |
-| | display_string | string | Coming soon | Coming soon |
-| | up_since | float | Coming soon | Coming soon |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **TLS** | **version** | string | Coming soon | Coming soon |
-| | cipher | string | Coming soon | Coming soon |
-| | curve | string | Coming soon | Coming soon |
-| | server_name | string | Coming soon | Coming soon |
-| | resumed | boolean | Coming soon | Coming soon |
-| | next_protocol | string | Coming soon | Coming soon |
-| | established | boolean | Coming soon | Coming soon |
-| | cert_chain_fuids | string | Coming soon | Coming soon |
-| | client_cert_chain_fuids | string | Coming soon | Coming soon |
-| | subject | string | Coming soon | Coming soon |
-| | issuer | string | Coming soon | Coming soon |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **SSH** | **version** | string | Coming soon | Coming soon |
-| | auth_success | boolean | Coming soon | Coming soon |
-| | client | string | Coming soon | Coming soon |
-| | server | string | Coming soon | Coming soon |
-| | cipher_algorithm | string | Coming soon | Coming soon |
-| | mac_algorithm | string | Coming soon | Coming soon |
-| | compression_algorithm | string | Coming soon | Coming soon |
-| | key_exchange_algorithm | string | Coming soon | Coming soon |
-| | host_key_algorithm | string | Coming soon | Coming soon |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **DHCP** | assigned_ip4 | bigint | Coming soon | Coming soon |
-| | mac | string | Coming soon | Coming soon |
-| | lease_time | double | Coming soon | Coming soon |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **IRC** | user | string | Coming soon | Coming soon |
-| | nickname | string | Coming soon | Coming soon |
-| | command | string | Coming soon | Coming soon |
-| | value | string | Coming soon | Coming soon |
-| | additional_data | string | Coming soon | Coming soon |
-| **Category** | **Attribute** | **Data Type** | **Description** | **Sample Values** |
-| **Flow** | in_packets | int | Coming soon | Coming soon |
-| | out_packets | int | Coming soon | Coming soon |
-| | **in_bytes** | int | Coming soon | Coming soon |
-| | **out_bytes** | int | Coming soon | Coming soon |
-| | conn_state | string | Coming soon | Coming soon |
-| | history | string | Coming soon | Coming soon |
-| | duration | float | Coming soon | Coming soon |
-| | src_os | string | Coming soon | Coming soon |
-| | dst_os | string | Coming soon | Coming soon |
-
-Note: It is not necessary to populate all of the attributes within the model.
-For attributes not populated in a single security event log/alert, contextual
-data may not be available. For example, the sample event below can be enriched
-with contextual data about the referenced endpoints (10.1.1.1 and
-192.168.10.10), but not a user, because username is not populated.
-
-> **date,time,source_ip,source_port,protocol,destination_ip,destination_port,bytes
-> 12/12/2015,23:14:56,10.1.1.1,1025,tcp,192.168.10.10,443,1183**
-
-**Context Models**
-==================
-
-The recommended approach for populating the context models (user, endpoint,
-network, etc.) involves consuming information from the systems most capable or
-providing the needed context. Populating the user context model is best
-accomplished by leveraging user/identity management systems such as Active
-Directory or Centrify and populating the model with details such as the user’s
-full name, job title, phone number, manager’s name, physical address,
-entitlements, etc. Similarly, an endpoint model can be populated by consuming
-information from endpoint/asset management systems (Tanium, Webroot, etc.),
-which provide information such as the services running on the system, system
-owner, business context, etc.
-
-**User Context Model**
-----------------------
-
-The data model for user context information is as follows:
-
-| **Attribute** | **Data Type** | **Description** | **Sample Values** |
-|------------------|------------------------------------------------------|--------------------------------------------------------------|-------------------------------------|
-| dvc_time | bigint | Timestamp from when the user context information is obtained | 1472653952 |
-| created | bigint | Timestamp from when user was created | 1472653952 |
-| Changed–––– | bigint | Timestamp from when user was updated | 1472653952 |
-| lastlogon | bigint | Timestamp from when user last logged on | 1472653952 |
-| logoncount | int | Number of times account has logged on | 232 |
-| lastreset | bigint | Timestamp from when user last reset passwod | 1472653952 |
-| expiration | bigint | Date/time when user expires | 1472653952 |
-| userid | string | Unique user id | 1234 |
-| username | string | Username in event log/alert | jsmith |
-| name_first | string | First name | John |
-| name_middle | string | Middle name | Henry |
-| name_last | string | Last name | Smith |
-| name_mgr | string | Manager’s name | Ronald Reagan |
-| phone | string | Phone number | 703-555-1212 |
-| email | string | Email address | jsmith\@company.com |
-| code | string | Job code | 3455 |
-| loc | string | Location | US |
-| departm | string | Department | IT |
-| dn | | Distinguished name | "CN=scm-admin-mej-test2-adk,OU=app- |
-| ou | string | Organizational unit | EAST |
-| empid | string | Employee ID | 12345 |
-| title | string | Job Title | Director of IT |
-| groups | string (comma separated list, no spaces after comma) | Groups to which the user belongs | “Domain Admins”, “Domain Users” |
-| dvc_type | string | Device type that generated the user context data | Active Directory |
-| dvc_vendor | string | Vendor | Microsoft |
-| dvc_version | string | Version | 8.1.2 |
-| additional_attrs | string | Additional attributes of user | Key value pairs |
-
-**Endpoint Context Model**
---------------------------
-
-The data model for endpoint context information is as follows:
-
-| **Abbreviation** | **Data Type** | **Description** | **Sample Values** |
-|------------------|--------------------------------------------------|------------------------------------------------------------------|------------------------------------------------------|
-| dvc_time | bigint | Timestamp from when the endpoint context information is obtained | 1472653952 |
-| ip4 | bigint | IP address of endpoint | Integer representaion of 10.1.1.1 |
-| ip6 | bigint | IP address of endpoint | Integer representaion of 10.1.1.1 |
-| os | string | Operating system | Redhat Linux 6.5.1 |
-| os_version | string | Version of OS | 5.4 |
-| os_sp | string | Service pack | SP 2.3.4.55 |
-| tz | string | timezone | EST |
-| hotfixes | string | Applied hotfixes | 993.2 |
-| disks | string | Available disks | \\Device\\HarddiskVolume1, \\Device\\HarddiskVolume2 |
-| removables | string | Removable media devices | USB Key |
-| nics | string | Network interfaces | fe10::28f4:1a47:658b:d6e8, fe82::28f4:1a47:658b:d6e8 |
-| drivers | string | Installed kernel drivers | ntoskrnl.exe, hal.dll |
-| users | string | Local user accounts | administrator, jsmith |
-| host | string | Hostname of endpoint | tes1.companya.com |
-| mac | string | MAC address of endpoint | fe10::28f4:1a47:658b:d6e8 |
-| owner | string | Endpoint owner (name) | John Smith |
-| vulns | string (comma separated, no spaces after commas) | Vulnerability identifiers (CVE identifier) | CVE-123, CVE-456 |
-| loc | string | Location | US |
-| departm | string | Department name | IT |
-| company | string | Company name | CompanyA |
-| regs | string (comma-separated) | Applicable regulations | HIPAA, SOX |
-| svcs | string (comma-separated) | Services running on system | Cisco Systems, Inc. VPN Service, Adobe LM Service |
-| procs | string | Processes | svchost.exe, sppsvc.exe |
-| criticality | string | Criticality of device | Very High |
-| apps | string (comma-separated) | Applications running on system | Microsoft Word, Chrome |
-| desc | string | Endpoint descriptor | Some string |
-| dvc_type | string | Device type that generated the log | Microsoft Windows 7 |
-| dvc_vendor | string | Vendor | Endgame |
-| dvc_version | string | Version | 2.1 |
-| architecture | string | CPU architecture | x86 |
-| uuid | string | Universally unique identifier | a59ba71e-18b0-f762-2f02-0deaf95076c6 |
-| memtotal | int | Total memory (bytes) | 844564433 |
-| additional_attrs | string | Additional attributes | Key value pairs |
-
-**VPN Context Model**
----------------------
-
-The data model for VPN context information is based on the VPN logs as follows:
-
-| **Abbreviation** | **Data Type** | **Description** | **Sample Values** |
-|------------------|-------------------------|----------------------------------------------------------------------------|------------------------------------------------------|
-| dvc_time | bigint | Timestamp from when the endpoint context information is obtained | 1472653952 |
-| ip4 | bigint | IP address of VPN box | Integer representaion of 10.1.1.1 |
-| ip6 | bigint | IP address of VPN box | Integer representaion of 10.1.1.1 |
-| vpn_vendor | string | Vendor VPN | Cisco |
-| vpn_version | string | Version VPN | 3.0 |
-| vpn_sp | string | VPN Service pack | 5 |
-| tz | string | VPN timezone | EST |
-| vpn_hotfixes | string | VPN Applied hotfixes | 1134 |
-| vpn_nics | string | Network interfaces | fe10::28f4:1a47:658b:d6e8, fe82::28f4:1a47:658b:d6e8 |
-| vpn_host | VPN Country Code | string | MX |
-| vpn_country_name | VPN Country Name | string | Mexico |
-| vpn_ip | | string | Integer representation of 10.1.1.2 |
-| vpn_encrypt | VPN encryption protocol | string | IPSEC |
-| vpn_username | string | VPN user account | jsmith |
-| vpn_user_ip | string | VPN User IP address | Integer representation of 10.1.1.2 |
-| vpn_user_cc | string | VPN Country Code | US |
-| vpn_user_cn | string | VPN Country Name | United States |
-| vpn_user_auth | string | VPN user authorization / role | Admin, normal user, etc |
-| vpn_account_vip | string | Criticality of the VPN account | Medium, High |
-| vpn_uuid | string | Universally unique identifier | a59ba71e-18b0-f762-2f02-0deaf95076c6 |
-| uuids | string | Universally unique identifier(s) comes from thee endpoint context if match | a59ba71e-18b0-f762-2f02-0deaf95xmexzA |
-| additional_attrs | string | Additional attributes | Key value pairs |
-
-**Network Context Model**
--------------------------
-
-The data model for network context information is based on “whois” information
-as follows:
-
-| **Attribute** | **Data Type** | **Description** | **Sample Values** |
-|----------------------------------------|---------------|----------------------------------------|-------------------|
-| domain_name | string | Domain name | |
-| registry_domain_id | string | Registry Domain ID | |
-| registrar_whois_server | string | Registrar WHOIS Server | |
-| registrar_url | string | Registrar URL | |
-| update_date | bigint | UTC timestamp | |
-| creation_date | bigint | Creation Date | |
-| registrar_registration_expiration_date | bigint | Registrar Registration Expiration Date | |
-| registrar | string | Registrar | |
-| registrar_iana_id | string | Registrar IANA ID | |
-| registrar_abuse_contact_email | string | Registrar Abuse Contact Email | |
-| registrar_abuse_contact_phone | string | Registrar Abuse Contact Phone | |
-| domain_status | string | Domain Status | |
-| registry_registrant_id | string | Registry Registrant ID | |
-| registrant_name | string | Registrant Name | |
-| registrant_organization | string | Registrant Organization | |
-| registrant_street | string | Registrant Street | |
-| registrant_city | string | Registrant City | |
-| registrant_state_province | string | Registrant State/Province | |
-| registrant_postal_code | string | Registrant Postal Code | |
-| registrant_country | string | Registrant Country | |
-| registrant_phone | string | Registrant Phone | |
-| registrant_email | string | Registrant Email | |
-| registry_admin_id | string | Registry Admin ID | |
-| name_server | string | Name Server | |
-| dnssec | string | DNSSEC | |
-
-### **Extensibility of Data Model**
-
-The aforementioned data model can be extended to accommodate custom attributes
-by embedding key-value pairs within the log/alert/context entries. Each model
-will support an additional attribute by the name of additional_attrs whose value
-would be a JSON string. This JSON string will contain a Map (and only a Map) of
-additional attributes that can’t be expressed in the specified model
-description. Regardless of the type of these additional attributes, they will
-always be interpreted as String. It’s up to the user, to translate them to
-appropriate types, if necessary, in the analytics layer. It is also the user’s
-responsibility to populate the aforementioned attribute as a Map, by presumably
-parsing out these attributes from the original message. For example, if a user
-wanted to extend the user context model to include a string attribute for “Desk
-Location” and “City”, the following string would be set for additional_attrs:
-
-| **Attribute Key** | **Attribute Value** |
-|-------------------|-------------------------------------------------|
-| additional_attrs | {"dsk_location":"B3-F2-W3", "city":"Palo Alto"} |
-
-Something similar can be done for endpoint context model, security event
-log/alert model and other entities.
-
-**Note:** This [UDF library](https://github.com/klout/brickhouse) can be used
-for converting to/from JSON.
-
-**Model Relationships**
------------------------
-
-The relationships between the data model entities are illustrated below.
-
-
-
-**Data Ingestion Framework**
-----------------------------
-
-One of the challenges in populating the data model is the large number of
-products and technologies that organizations are currently using to manage
-security event logs/alerts, user and endpoint information. There are literally
-dozens of vendors in each category that offer technologies that could be used to
-populate the model. The labor required to transform the data and map the
-attributes to the data model is extensive when you consider how many
-technologies are in the mix at each organization (and across organizations). One
-way to address this challenge is with a Data Ingestion Framework that provides a
-configuration-based mechanism to perform the transformations and mappings. A
-configuration-based capability will allow the ingest pipelines to become
-portable and reusable across the community. For example, if I create an ingest
-pipeline for Centrify to populate the user context model, it can be shared with
-other users of Centrify who can immediately realize the benefit. Such a
-framework could allow the community to quickly build the necessary pipelines for
-the dozens (and hundreds) of technologies being used in the market. Without a
-standard ingest framework, each pipeline is built independently, requiring more
-labor, providing no standardization and little portability. It’s also important
-that the data ingestion framework support the ability to both capture the “raw”
-event and create a meta event that represents the normalized event and maps the
-attributes to the defined data model. This will ensure both stream and batch
-processing use cases are supported.
-
-Streamsets is an ingest framework that provides the needed functionality
-outlined above. Sample Streamsets ingest pipelines for populating the ODM with
-common data sources will be published to the Spot Github repo.
-
-**Data Formats**
-----------------
-
-**Avro**
---------
-
-Avro is the recommended data format due to its schema representation,
-compatibility checks, and interoperability with Hadoop. Avro supports a pure
-JSON representation for readability and ease of use but also a binary
-representation of the data for efficient storage. Avro is the optimal format for
-streaming-based analytic use cases. A sample event and corresponding schema
-representation are detailed below.
-
-**Event**
-
-{
-
-"eventtime":1469562994,
-
-"src_ip4":”192.168.1.1”,
-
-“src_host”:”test1.clouera.com”,
-
-“src_port”:1029, “dst_ip4”:”192.168.21.22”,
-
-“dst_host”:”test3.companyA.com”,
-
-“dst_port”:443,
-
-“dvc_type”:”sshd”,
-
-“category”:”auth”,
-
-“aproto”:”sshd”,
-
-“msg”:”user:mhicks successfully logged in to test3.companyA.com from
-192.168.1.1”,
-
-“username”:”mhicks”,
-
-“Severity”:3,
-
-}
-
-
-
-**Schema**
-
-{
-
-"type": "record",
-
-"doc":"This event records SSHD activity",
-
-"name": "auth",
-
-"fields":{
-
-{"name":"eventtime", "type":"long", "doc":"Stop time of event""},
-
-{"name":"src_ip4", "type":"long", "doc":”Source IP Address"},
-
-{"name":"src_host", "type":"string",”doc”:”Source hostname},
-
-{"name":"src_port", "type":"int",”doc”:”Source port”},
-
-{"name":"dst_ip4", "type":"long", "doc"::”Destination IP Address"},
-
-{"name":"dst_host", "type":"string", "doc":”Destination IP Address"},
-
-{"name":"dst_port", "type":"int", ”doc”:”Destination port”},
-
-{"name":"dvc_type", "type":"string", “doc”:”Source device type”},
-
-{"name":"category", "type":"string",”doc”:”category/type of event message”},
-
-{"name":"aproto", "type":"string",”doc”:”Application or network protocol”},
-
-{"name":"msg", "type":"string",”doc”:”event message”},
-
-{"name":"username", "type":"string",”doc”:”username”},
-
-{"name":"severity", "type":"int",”doc”:”severity of event on scale of 1-10”},
-
-}
-
-
-
-**JSON**
---------
-
-JSON is commonly used as a data-interchange format due to it’s ease of use and
-familiarity within the development community. The corresponding JSON object for
-the sample event described previously is noted below.
-
-{
-
-“eventtime”:1469562994,
-
-“src_ip4”:”192.168.1.1”,
-
-“src_host”:”test1.clouera.com”,
-
-“src_port”:1029,
-
-“dst_ip4”:”192.168.21.22”,
-
-“dst_host”:”test3.companyA.com”,
-
-“dst_port”:443,
-
-“aproto”:”sshd”,
-
-“msg”:”user:mhicks successfully logged in to test3.companyA.com from
-192.168.1.1”,
-
-“username”:”mhicks”,
-
-}
-
-**Parquet**
------------
-
-Parquet is a columnar storage format that offers the benefits of compression and
-efficient columnar data representation and is optimal for batch analytic use
-cases. More information on parquet can be found here:
-https://parquet.apache.org/documentation/latest/ It should be noted that
-conversion from Avro to Parquet is supported. This allows for data collected and
-analyzed for stream-based use cases to be easily converted to Parquet for
-longer-term batch analytics.
-
-**Example - Advanced Threat Modeling**
---------------------------------------
-
-In this example, the ODM is leveraged to build an “event” table for a threat
-model that uses attributes native to the ODM and derived attributes, which are
-calculations based on the aggregate data stored in the model. In this context,
-an “event” table is defined by the attributes to be evaluated for predictive
-power in identifying threats and the actual attribute values (i.e rows in the
-table). In the example below, the event table is composed of the following
-attributes, which are then leveraged to identify threats via a Risk Score
-analytic model:
-
-- “src_ipv4” - This attribute is native to the security event log component of
- the ODM and represents the source IP address of the corresponding table row
-
-- “os” - This attribute is native to the endpoint context component of the ODM
- and represents the operating system of the endpoint system in the table row
-
-- SUM (in_bytes + out_bytes) for the last 7 days - “in_bytes” and “out_bytes”
- are native to the security event log component of the ODM. This derived
- attribute represents a summation of bytes between the source address and
- destination domain over the last 7 days
-
-- “dst_domain” - This attribute is native to the security event log component
- of the ODM and represents the destination domain
-
-- Days since “creation_date” - “creation_date” is native to the network
- context component of the ODM and represents the date the referenced domain
- was registered. This derived attribute calculates the days since the domain
- was created/registered.
-
-| **src_ipv4** | **OS** | **dst domain** | **Days since “creation_date”** | **SUM (in_bytes + out_bytes)** | **Risk Score (1-100)** |
-|
<TRUNCATED>
[03/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/GraphQLStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/GraphQLStore.js b/spot-oa/ui/js/stores/GraphQLStore.js
new file mode 100644
index 0000000..ad0989e
--- /dev/null
+++ b/spot-oa/ui/js/stores/GraphQLStore.js
@@ -0,0 +1,89 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+const $ = require('jquery');
+const SpotConstants = require('../constants/SpotConstants');
+
+class GraphQLStore {
+ constructor() {
+ this.variables = {};
+ this.data = {};
+ }
+
+ getQuery() {
+ return null;
+ }
+
+ setVariable(name, value) {
+ this.variables[name] = value;
+ }
+
+ getVariable(name) {
+ return this.variables[name];
+ }
+
+ unsetVariable(name) {
+ delete this.variables[name];
+ }
+
+ setData(data) {
+ this.data = data;
+ }
+
+ getData() {
+ if (Object.keys(this.data).length==0 || this.data.loading || this.data.error) return this.data;
+
+ return {loading: false, data: this.unboxData(this.data)};
+ }
+
+ resetData() {
+ this.setData({});
+ }
+
+ sendQuery() {
+ const query = this.getQuery();
+ const variables = this.variables;
+
+ this.setData({loading: true});
+ $.post({
+ accept: 'application/json',
+ contentType: 'application/json',
+ dataType: 'json',
+ data: JSON.stringify({
+ query,
+ variables
+ }),
+ url: SpotConstants.GRAPHQL_ENDPOINT
+ })
+ .done((response) => {
+ if (response.errors) {
+ console.error('Unexpected GraphQL error', response)
+ this.setData({error: 'Oops... something went wrong'});
+ }
+ else {
+ this.setData(response.data);
+ if(response.data !== undefined && JSON.stringify(response.data).indexOf('"success":true') !== -1)
+ this.reloadElements();
+ }
+ })
+ .fail((jqxhr, textStatus, error) => {
+ console.error('Unexpected GraphQL error', jqxhr.responseJSON)
+ this.setData({error: `${textStatus}: ${error}`})
+ });
+ }
+}
+
+module.exports = GraphQLStore;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/IngestSummaryStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/IngestSummaryStore.js b/spot-oa/ui/js/stores/IngestSummaryStore.js
index 0f3d4a9..ac2a6a2 100755
--- a/spot-oa/ui/js/stores/IngestSummaryStore.js
+++ b/spot-oa/ui/js/stores/IngestSummaryStore.js
@@ -1,178 +1,116 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-const assign = require('object-assign');
-const d3 = require('d3');
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotDispatcher = require('../dispatchers/SpotDispatcher');
const SpotConstants = require('../constants/SpotConstants');
-const DateUtils = require('../utils/DateUtils');
-const RestStore = require('../stores/RestStore');
-
-const PIPELINE_FILTER = 'pipeline';
-const CURRENT_YEAR_FILTER = 'year';
-const CURRENT_MONTH_FILTER = 'month';
-
-const requestQueue = [];
-const requestErrors = [];
-
-const IngestSummaryStore = assign(new RestStore(SpotConstants.API_INGEST_SUMMARY), {
- PIPELINES: {
- [SpotConstants.PIPELINE_NETFLOW]: 'Netflow',
- [SpotConstants.PIPELINE_DNS]: 'Dns',
- [SpotConstants.PIPELINE_PROXY]: 'Proxy'
- },
- errorMessages: {
- 404: 'No details available'
- },
- setStartDate(date) {
- this._startDate = date;
- },
- getStartDate() {
- return this._startDate;
- },
- setEndDate(date) {
- this._endDate = date;
- },
- getEndDate() {
- return this._endDate;
- },
- setPipeline(pipeline) {
- this.setRestFilter(PIPELINE_FILTER, pipeline);
- },
- getPipeline() {
- return this.getRestFilter(PIPELINE_FILTER);
- },
- setCurrentDate(date) {
- this.setRestFilter(CURRENT_YEAR_FILTER, date.getFullYear())
-
- const month = date.getMonth() + 1 + "";
- this.setRestFilter(CURRENT_MONTH_FILTER, month.length==1 ? `0${month}`:month);
-
- this._currentDate = date;
- },
- getCurrentDate() {
- return this._currentDate;
- },
- /**
- * Start asking the server for CSV data to create the chart
- **/
- requestSummary: function () {
- var startDate, endDate, date, delta, startRequests, i, month;
-
- startDate = DateUtils.parseDate(this.getStartDate());
- endDate = DateUtils.parseDate(this.getEndDate());
-
- // Find out how many request need to be made
- delta = (endDate.getFullYear() - startDate.getFullYear()) * 12 + (endDate.getMonth() - startDate.getMonth());
-
- startRequests = requestQueue.length == 0;
-
- // Go to first day in month
- date = new Date(startDate);
- date.setDate(1);
-
- // Queue date requests
- requestQueue.push(date);
- for (i = 1; i <= delta; i++) {
- requestQueue.push(DateUtils.calcDate(date, i, 'month'));
- }
-
- // dequeue is no request is running
- startRequests && this.dequeue();
- },
- dequeue: function () {
- if (requestQueue.length == 0) return;
-
- const date = requestQueue.shift();
- this.setCurrentDate(date);
-
- this.reload();
- },
- setData: function (data) {
- var startDate, endDate, date, dayFilter, parse;
-
- // Does the loading indicator needs to be displayed?
- if (data.loading) {
- if (!this._data.loading) {
- this._data = data;
- this.emitChangeData();
- }
- // Do nothing when loading is in progress
- return;
- }
-
- // Store errors for later usage
- if (data.error) {
- requestErrors.push(data);
- }
- else if (data.data) {
- parse = d3.time.format("%Y-%m-%d %H:%M:%S%Z").parse; // Date formatting parser
- startDate = DateUtils.parseDate(this.getStartDate());
- endDate = DateUtils.parseDate(this.getEndDate());
- date = DateUtils.parseDate(this.getCurrentDate());
-
- if (date.getFullYear() == startDate.getFullYear() && date.getMonth() == startDate.getMonth()) {
- dayFilter = startDate.getDate();
- data.data = data.data.filter(function (row) {
- return DateUtils.parseDate(row.date, true).getDate() >= dayFilter
- });
- }
+const ObservableGraphQLStore = require('./ObservableGraphQLStore');
- if (date.getFullYear() == endDate.getFullYear() && date.getMonth() == endDate.getMonth()) {
- dayFilter = endDate.getDate();
- data.data = data.data.filter(function (row) {
- return DateUtils.parseDate(row.date, true).getDate() <= dayFilter
- });
+const START_DATE_VAR = 'startDate';
+const END_DATE_VAR = 'endDate';
+
+class IngestSummaryStore extends ObservableGraphQLStore {
+ constructor() {
+ super();
+
+ this.pipeline = null;
+ this.PIPELINES = {};
+ this.PIPELINES[SpotConstants.PIPELINE_NETFLOW] = 'Netflow';
+ this.PIPELINES[SpotConstants.PIPELINE_DNS] = 'Dns';
+ this.PIPELINES[SpotConstants.PIPELINE_PROXY] = 'Proxy';
+ }
+
+ getQuery() {
+ let pipeline = this.getPipeline() || '';
+
+ return `
+ query($startDate:SpotDateType!,$endDate:SpotDateType!) {
+ ${pipeline} {
+ ingestSummary(startDate:$startDate, endDate:$endDate) {
+ date: datetime
+ total
+ }
+ }
}
+ `;
+ }
- // Parse dates and numbers.
- data.data.forEach(function (d) {
- d.date = parse(`${d.date}:00-0000`);
- d.total = +d.total;
- });
+ unboxData(data) {
+ let pipeline = this.getPipeline();
+ let parser = parser = d3.time.format("%Y-%m-%d %H:%M:%S%Z").parse;
- // Sort the data by date ASC
- data.data.sort(function (a, b) {
- return a.date - b.date;
- });
+ let dataByMonth = {};
+ data[pipeline].ingestSummary.forEach(record => {
+ record = {total: record.total, date: parser(`${record.date}-0000`)};
+ let month = record.date.toISOString().substr(0,7);
- if (!this._data.data) this._data.data = [];
+ if (!(month in dataByMonth)) dataByMonth[month] = [];
+ dataByMonth[month].push(record);
+ });
- this._data.data.push(data.data);
- }
+ return Object.keys(dataByMonth).map(month => dataByMonth[month].sort((a, b) => a.date - b.date));
+ }
- this._data.loading = requestQueue.length > 0;
+ setStartDate(date) {
+ this.setVariable(START_DATE_VAR, date);
+ }
- if (!this._data.loading) {
- if (this._data.data && this._data.data.length==0) {
- // Broadcast first found error
- this._data = requestErrors[0];
- }
- this.emitChangeData();
- }
- else {
- setTimeout(this.dequeue.bind(this), 1);
- }
+ getStartDate() {
+ return this.getVariable(START_DATE_VAR);
}
-});
+
+ setEndDate(date) {
+ this.setVariable(END_DATE_VAR, date);
+ }
+
+ getEndDate() {
+ return this.getVariable(END_DATE_VAR);
+ }
+
+ setPipeline(pipeline) {
+ this.pipeline = pipeline;
+ }
+
+ getPipeline() {
+ return this.pipeline;
+ }
+}
+
+const iss = new IngestSummaryStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
switch (action.name) {
case SpotConstants.START_DATE:
- IngestSummaryStore.setStartDate(action.date);
+ iss.setStartDate(action.date);
break;
case SpotConstants.END_DATE:
- IngestSummaryStore.setEndDate(action.date);
+ iss.setEndDate(action.date);
break;
}
break;
case SpotConstants.RELOAD_INGEST_SUMMARY:
- IngestSummaryStore.requestSummary();
+ iss.sendQuery();
break;
+ case SpotConstants.UPDATE_PIPELINE:
+ iss.setPipeline(action.pipeline)
+ break;
}
});
-module.exports = IngestSummaryStore;
+module.exports = iss;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/JsonStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/JsonStore.js b/spot-oa/ui/js/stores/JsonStore.js
index 2c7fa74..324c727 100755
--- a/spot-oa/ui/js/stores/JsonStore.js
+++ b/spot-oa/ui/js/stores/JsonStore.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const $ = require('jquery');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/ObservableGraphQLStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/ObservableGraphQLStore.js b/spot-oa/ui/js/stores/ObservableGraphQLStore.js
new file mode 100644
index 0000000..7d9ef1c
--- /dev/null
+++ b/spot-oa/ui/js/stores/ObservableGraphQLStore.js
@@ -0,0 +1,55 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+const EventEmitter = require('events').EventEmitter;
+
+const GraphQLStore = require('./GraphQLStore');
+
+const DATA_CHANGE_EVENT = 'data-change';
+
+class ObervableGraphQLStore extends GraphQLStore {
+ constructor() {
+ super();
+ this.eventEmitter = new EventEmitter();
+ }
+
+ addChangeDataListener(callback) {
+ this.addListener(DATA_CHANGE_EVENT, callback);
+ }
+
+ removeChangeDataListener(callback) {
+ this.removeListener(DATA_CHANGE_EVENT, callback);
+ }
+
+ addListener(eventName, callback) {
+ this.eventEmitter.on(eventName, callback);
+ }
+
+ removeListener(eventName, callback) {
+ this.eventEmitter.removeListener(eventName, callback);
+ }
+
+ notifyListeners(eventName) {
+ this.eventEmitter.emit(eventName);
+ }
+
+ setData(data) {
+ super.setData(data);
+ this.notifyListeners(DATA_CHANGE_EVENT);
+ }
+}
+
+module.exports = ObervableGraphQLStore;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/ObservableWithHeadersGraphQLStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/ObservableWithHeadersGraphQLStore.js b/spot-oa/ui/js/stores/ObservableWithHeadersGraphQLStore.js
new file mode 100644
index 0000000..679b46d
--- /dev/null
+++ b/spot-oa/ui/js/stores/ObservableWithHeadersGraphQLStore.js
@@ -0,0 +1,31 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+const ObservableGraphQLStore = require('./ObservableGraphQLStore');
+
+class ObservableWithHeadersGraphQLStore extends ObservableGraphQLStore {
+ getData() {
+ const data = super.getData();
+
+ if (data.loading || data.error) return data;
+
+ data.headers = this.headers;
+
+ return data;
+ }
+}
+
+module.exports = ObservableWithHeadersGraphQLStore
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/RestStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/RestStore.js b/spot-oa/ui/js/stores/RestStore.js
index 16e8203..6c9b797 100755
--- a/spot-oa/ui/js/stores/RestStore.js
+++ b/spot-oa/ui/js/stores/RestStore.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var $ = require('jquery');
var assign = require('object-assign');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/SpotStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/SpotStore.js b/spot-oa/ui/js/stores/SpotStore.js
index 488f4dc..7912ebb 100755
--- a/spot-oa/ui/js/stores/SpotStore.js
+++ b/spot-oa/ui/js/stores/SpotStore.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var EventEmitter = require('events').EventEmitter;
var assign = require('object-assign');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/__tests__/JsonStore.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/__tests__/JsonStore.test.js b/spot-oa/ui/js/stores/__tests__/JsonStore.test.js
index 1a0eb81..19695e6 100644
--- a/spot-oa/ui/js/stores/__tests__/JsonStore.test.js
+++ b/spot-oa/ui/js/stores/__tests__/JsonStore.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
jest.mock('jquery');
const $ = require('jquery');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/__tests__/RestStore.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/__tests__/RestStore.test.js b/spot-oa/ui/js/stores/__tests__/RestStore.test.js
index 8a2e4cd..63b8145 100644
--- a/spot-oa/ui/js/stores/__tests__/RestStore.test.js
+++ b/spot-oa/ui/js/stores/__tests__/RestStore.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
jest.mock('jquery');
const $ = require('jquery');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/stores/__tests__/SpotStore.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/stores/__tests__/SpotStore.test.js b/spot-oa/ui/js/stores/__tests__/SpotStore.test.js
index 67ec57a..5788284 100644
--- a/spot-oa/ui/js/stores/__tests__/SpotStore.test.js
+++ b/spot-oa/ui/js/stores/__tests__/SpotStore.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
const SpotStore = require('../SpotStore');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/tooltip.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/tooltip.js b/spot-oa/ui/js/tooltip.js
index edc4b6f..dc53183 100755
--- a/spot-oa/ui/js/tooltip.js
+++ b/spot-oa/ui/js/tooltip.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
$(function () {
$('body').tooltip({
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/utils/CategoryLayout.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/utils/CategoryLayout.js b/spot-oa/ui/js/utils/CategoryLayout.js
index 7ac0d5c..28b9829 100755
--- a/spot-oa/ui/js/utils/CategoryLayout.js
+++ b/spot-oa/ui/js/utils/CategoryLayout.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
class CategoryLayout {
constructor() {
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/utils/DateUtils.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/utils/DateUtils.js b/spot-oa/ui/js/utils/DateUtils.js
index 82be435..a0978cc 100644
--- a/spot-oa/ui/js/utils/DateUtils.js
+++ b/spot-oa/ui/js/utils/DateUtils.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var DateUtils = {
/**
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/utils/SpotUtils.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/utils/SpotUtils.js b/spot-oa/ui/js/utils/SpotUtils.js
index 0549c74..953f5e1 100755
--- a/spot-oa/ui/js/utils/SpotUtils.js
+++ b/spot-oa/ui/js/utils/SpotUtils.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const Base64 = require('js-base64').Base64;
@@ -114,6 +129,41 @@ var SpotUtils = {
},
decodeId(id) {
return Base64.decode(id.replace(ID_REPLACEMENT_REGEX, ID_REPLACE));
+ },
+ filterTextOnSelect(selectionEl, str, isCaseSensitive) {
+ if (isCaseSensitive)
+ str = str.toLowerCase();
+ // cache the jQuery object of the <select> element
+ var $el = $(selectionEl);
+ if (!$el.data("options")) {
+ // cache all the options inside the <select> element for easy recover
+ $el.data("options", $el.find("option").clone());
+ }
+ var newOptions = $el.data("options").filter(function () {
+ var text = $(this).text();
+ if (isCaseSensitive)
+ text = text.toLowerCase();
+ //select the first element of all results
+ if ($el.children().length > 0)
+ $el.children().get(0).selected = true;
+ return text.match(str);
+ });
+ $el.empty().append(newOptions);
+ },
+ switchDivs(elOne, elTwo) {
+ let elOneVal = $(`#${elOne}`).attr('style').split(';')[0].split(' ')[1] || '';
+ let elTwoVal = $(`#${elTwo}`).attr('style').split(';')[0].split(' ')[1] || '';
+
+ if (elOneVal === elTwoVal) {
+ $(`#${elOne}`).css({'order': '1'});
+ $(`#${elTwo}`).css({'order': '2'});
+ } else if(elOneVal === '1') {
+ $(`#${elOne}`).css({'order': '2'});
+ $(`#${elTwo}`).css({'order': '1'});
+ } else {
+ $(`#${elOne}`).css({'order': '1'});
+ $(`#${elTwo}`).css({'order': '2'});
+ }
}
};
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/utils/__tests__/CategoryLayout.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/utils/__tests__/CategoryLayout.test.js b/spot-oa/ui/js/utils/__tests__/CategoryLayout.test.js
index 16dbc68..168926f 100644
--- a/spot-oa/ui/js/utils/__tests__/CategoryLayout.test.js
+++ b/spot-oa/ui/js/utils/__tests__/CategoryLayout.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
const CategoryLayout = require('../CategoryLayout');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/utils/__tests__/DateUtils.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/utils/__tests__/DateUtils.test.js b/spot-oa/ui/js/utils/__tests__/DateUtils.test.js
index a622f0c..2fc255b 100644
--- a/spot-oa/ui/js/utils/__tests__/DateUtils.test.js
+++ b/spot-oa/ui/js/utils/__tests__/DateUtils.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
const DateUtils = require('../DateUtils');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/js/utils/__tests__/SpotUtils.test.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/utils/__tests__/SpotUtils.test.js b/spot-oa/ui/js/utils/__tests__/SpotUtils.test.js
index 9ccd8cd..30bea82 100644
--- a/spot-oa/ui/js/utils/__tests__/SpotUtils.test.js
+++ b/spot-oa/ui/js/utils/__tests__/SpotUtils.test.js
@@ -1,4 +1,18 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
const DateUtils = require('../DateUtils');
const SpotUtils = require('../SpotUtils');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/package.json b/spot-oa/ui/package.json
index 3863c64..3a61cf6 100644
--- a/spot-oa/ui/package.json
+++ b/spot-oa/ui/package.json
@@ -2,7 +2,7 @@
"name": "spot_ui",
"version": "1.1.0",
"description": "Apache Spot :: UI",
- "repository": "https://github.com/Open-Network-Insight/open-network-insight",
+ "repository": "https://github.com/apache/incubator-spot",
"license": "Apache License, Version 2.0",
"dependencies": {
"bootstrap": "3.3.5",
@@ -19,6 +19,7 @@
"object-assign": "^1.0.0",
"react": "^15.3.2",
"react-dom": "^15.3.2",
+ "sweetalert2": "^6.6.0",
"underscore": "^1.8.3"
},
"devDependencies": {
@@ -36,7 +37,7 @@
"scripts": {
"test": "jest",
"postinstall": "npm run build-all",
- "watch-ingest-summary": "NODE_ENV=development watchify js/ingest-summary.js -o js/ingest-summary.bundle.min.js -v -d",
+ "watch-ingest-summary": "watchify js/ingest-summary.js -o js/ingest-summary.bundle.min.js -v",
"build-all": "npm run build-flow && npm run build-dns && npm run build-proxy && npm run build-ingest-summary",
"build-flow": "cd flow/ && npm run build-all && cd ../",
"build-dns": "cd dns/ && npm run build-all && cd ../",
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/ipython_notebook.html b/spot-oa/ui/proxy/ipython_notebook.html
new file mode 100644
index 0000000..01dde73
--- /dev/null
+++ b/spot-oa/ui/proxy/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>Proxy :: Advanced mode</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Font awesome ICONS-->
+ <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+
+ <style>
+ .spot-row {
+ height: 100%;
+ }
+
+ td.srcIP_rep, td.dstIP_rep {
+ white-space: nowrap;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: Proxy :: Advanced mode</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+ <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/components/DetailsPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/components/DetailsPanel.react.js b/spot-oa/ui/proxy/js/components/DetailsPanel.react.js
index 6e5a93e..0fc9e76 100755
--- a/spot-oa/ui/proxy/js/components/DetailsPanel.react.js
+++ b/spot-oa/ui/proxy/js/components/DetailsPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
@@ -10,9 +25,6 @@ var DetailsPanel = React.createClass({
mixins: [GridPanelMixin, DetailsGridMixin],
store: DetailsStore,
// Custom cells
- _render_p_date_cell: function (date, item) {
- return date + ' ' + item.p_time;
- },
_render_host_cell(host) {
return (
<p className="spot-text-wrapper" data-toggle="tooltip">
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/components/FilterInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/components/FilterInput.react.js b/spot-oa/ui/proxy/js/components/FilterInput.react.js
index 666fc3e..3599603 100755
--- a/spot-oa/ui/proxy/js/components/FilterInput.react.js
+++ b/spot-oa/ui/proxy/js/components/FilterInput.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/components/IncidentProgressionPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/components/IncidentProgressionPanel.react.js b/spot-oa/ui/proxy/js/components/IncidentProgressionPanel.react.js
index 67adb5e..3dc4ffb 100755
--- a/spot-oa/ui/proxy/js/components/IncidentProgressionPanel.react.js
+++ b/spot-oa/ui/proxy/js/components/IncidentProgressionPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const $ = require('jquery');
const d3 = require('d3');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/components/NetworkViewPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/components/NetworkViewPanel.react.js b/spot-oa/ui/proxy/js/components/NetworkViewPanel.react.js
index 7862c63..7bc4035 100755
--- a/spot-oa/ui/proxy/js/components/NetworkViewPanel.react.js
+++ b/spot-oa/ui/proxy/js/components/NetworkViewPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const $ = require('jquery');
const assign = require('object-assign');
@@ -102,7 +117,7 @@ var NetworkViewPanel = React.createClass({
// Delete old links
this.link.exit().remove();
- // Update nodes
+ // Update nodesw
this.node = this.canvas.selectAll('.node, .proxy_node')
.data(nodes.filter((node) => node.visible), function(d) { return d.id; });
@@ -261,7 +276,7 @@ var NetworkViewPanel = React.createClass({
type: 'Root',
tooltip: 'Double click to toggle child nodes',
rep: -1,
- visible: true,
+ visible: state.data.length > 0 ? true : false,
expanded: false,
root: true
};
@@ -375,7 +390,7 @@ var NetworkViewPanel = React.createClass({
0
);
} else {
- node.size = node.hits.length;
+ node.size = node.hits === undefined ? 0 : node.hits.length;
}
nodes.push(node);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/components/ScoreNotebook.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/components/ScoreNotebook.react.js b/spot-oa/ui/proxy/js/components/ScoreNotebook.react.js
new file mode 100644
index 0000000..80dc3bd
--- /dev/null
+++ b/spot-oa/ui/proxy/js/components/ScoreNotebook.react.js
@@ -0,0 +1,199 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var SuspiciousStore = require('../stores/SuspiciousStore');
+var NotebookStore = require('../stores/NotebookStore');
+var EdInActions = require('../../../js/actions/EdInActions');
+var SelectInput = require('../../../js/components/scoring/SelectInput.react');
+var FilterSelectInput = require('../../../js/components/scoring/FilterSelectInput.react');
+var ButtonsScoring = require('../../../js/components/scoring/ButtonsScoring.react');
+var RatingInput = require('../../../js/components/scoring/RatingInput.react');
+var SearchGlobalInput = require('../../../js/components/scoring/SearchGlobalInput.react');
+var ScoreMessage = require('../../../js/components/scoring/ScoreMessage.react');
+
+var SpotUtils = require('../../../js/utils/SpotUtils');
+
+var ScoreNotebook = React.createClass({
+ // mixins: [GridPanelMixin],
+ emptySetMessage: 'There is no data available for selected date.',
+ propTypes: {
+ date: React.PropTypes.string.isRequired,
+ },
+ getInitialState: function () {
+ return {
+ scoredEmelents: []
+ };
+ },
+ componentDidMount: function() {
+ SuspiciousStore.addChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ this.setState({size: NotebookStore.completeClass})
+ },
+ componentWillUnmount: function () {
+ SuspiciousStore.removeChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ },
+ render: function () {
+ var content, state, data, cssCls, uriArr = [];
+ state = this.state || {};
+
+ if (state.error)
+ {
+ content = (
+ <div className="text-center text-danger">
+ {state.error}
+ </div>
+ );
+ }
+ else if (state.loading)
+ {
+ content = (
+ <div className="spot-loader">
+ Loading <span className="spinner"></span>
+ </div>
+ );
+ }
+ else if (!state.data || state.data.length === 0)
+ {
+ content = (
+ <div className="text-center">
+ {this.emptySetMessage || ''}
+ </div>
+ );
+ }
+ else
+ {
+ state.data.map((obj) => {
+ if(uriArr.indexOf(obj.fulluri) === -1) {
+ uriArr.push(obj.fulluri);
+ }
+ });
+
+ data = [
+ {value: 1, name: 'High', radioName: 'optradio', selected: true},
+ {value: 2, name: 'Medium', radioName: 'optradio', selected: false},
+ {value: 3, name: 'Low', radioName: 'optradio', selected: false}
+ ];
+
+ content = (
+ <div>
+ <div className="margin-up-down">
+ <SearchGlobalInput col="6" maxlength="255"/>
+ <RatingInput data={data} col="6"/>
+ </div>
+ <div className="margin-up-down">
+ <ButtonsScoring name="Score" action="score" onChange={this.score.bind(this)} col="3"/>
+ <ButtonsScoring name="Save" action="save" onChange={this.save.bind(this)} col="3"/>
+ <ButtonsScoring name="Reset Scoring" action="reset" onChange={this.reset.bind(this)} col="3"/>
+ </div>
+ <div className="margin-up-down">
+ <FilterSelectInput nameBox="URI..." idSelect="#fullUri" idInput="fullUriIn" col="9"/>
+ </div>
+ <div className="margin-up-down">
+ <SelectInput title="Source IP" who="fullUri" options={uriArr} col="9"/>
+ </div>
+ <div className="margin-up-down">
+ <ScoreMessage who="scoreMsg"/>
+ </div>
+ </div>
+ );
+ }
+ cssCls = this.state.size ? 'col-md-6 col-lg-6 col-xs-12' : 'col-md-offset-3 col-lg-offset-3 col-md-6 col-lg-6 col-xs-12';
+
+ return(
+ <div className={cssCls + ' spot-frame'}>
+ {content}
+ </div>
+ )
+ },
+ _onChange: function() {
+ const data = SuspiciousStore.getData();
+ this.setState(data);
+ },
+ reset: function() {
+ swal({
+ title: 'Are you sure?',
+ text: "You won't be able to revert this!",
+ type: 'warning',
+ showCancelButton: true,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Yes, reset all!'
+ }).then(() => {
+ EdInActions.resetScoring(SpotUtils.getCurrentDate());
+ swal({
+ title: 'Done!',
+ text: "All scores have been reset.",
+ type: 'success',
+ showCancelButton: false,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Ok!'
+ }).then(() => {
+ this.setState({loading: true});
+ });
+ });
+ },
+ save: function() {
+ let variables = [];
+
+ if (this.state.scoredEmelents.length === 0) {
+ swal('Warning.','You should score at least 1 threat.','warning');
+ } else {
+ this.state.scoredEmelents.map((row) => {
+ variables.push({
+ 'date': SpotUtils.getCurrentDate(),
+ 'uri': row[0] || "None",
+ 'score': row[1],
+ });
+ });
+
+ EdInActions.saveScoring(variables);
+ $('#scoreMsg').addClass("hidden");
+ this.setState({scoredEmelents: [], loading: true});
+ }
+ },
+ score: function() {
+ //this should be changed to take all data at the time, time needed.
+ let dataScored = this.state.scoredEmelents || [];
+ let quickIpScoring = document.getElementById('globalTxt').value;
+ let fullUri = document.getElementById('fullUri').value;
+ let rating = $('input[name="optradio"]:checked').val();
+
+ //callback from the father
+ if(quickIpScoring !== '') {
+ dataScored.push([quickIpScoring, rating]);
+ } else {
+ dataScored.push([fullUri, rating]);
+ }
+
+ this.removeSelected([fullUri, quickIpScoring]);
+ $('#scoreMsg').removeClass("hidden");
+ this.setState({scoredEmelents: dataScored});
+ },
+ removeSelected: function(data) {
+ //when an user score, all elements selected need to be removed.
+ data.map((element) => data.map((e) => e !== '' ? $(`option[value="${e}"]`).remove() : ''));
+ $(".select-picker, #globalTxt").val('');
+ }
+
+});
+
+
+module.exports = ScoreNotebook;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/components/SuspiciousPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/components/SuspiciousPanel.react.js b/spot-oa/ui/proxy/js/components/SuspiciousPanel.react.js
index 95a9024..cdc6197 100755
--- a/spot-oa/ui/proxy/js/components/SuspiciousPanel.react.js
+++ b/spot-oa/ui/proxy/js/components/SuspiciousPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
const ReactDOMServer = require('react-dom/server');
@@ -61,9 +76,6 @@ var SuspiciousPanel = React.createClass({
</span>
);
},
- _render_p_date_cell: function (date, item) {
- return date + ' ' + item['p_time'];
- },
_render_host_cell: function (host, item, idx) {
var reps, highestRep;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/components/TimelinePanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/components/TimelinePanel.react.js b/spot-oa/ui/proxy/js/components/TimelinePanel.react.js
index 01dcb41..33bc725 100755
--- a/spot-oa/ui/proxy/js/components/TimelinePanel.react.js
+++ b/spot-oa/ui/proxy/js/components/TimelinePanel.react.js
@@ -1,5 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react') ;
const TimelineStore = require('../stores/TimelineStore');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/constants/ProxyConstants.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/constants/ProxyConstants.js b/spot-oa/ui/proxy/js/constants/ProxyConstants.js
deleted file mode 100755
index ed19357..0000000
--- a/spot-oa/ui/proxy/js/constants/ProxyConstants.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var ProxyConstants = {
- // API URLS
- API_SUSPICIOUS: '../../data/proxy/${date}/proxy_scores.tsv',
- API_DETAILS: '../../data/proxy/${date}/edge-${clientip}-${hash}.tsv',
- API_COMMENTS: '../../data/proxy/${date}/threats.csv',
- API_INCIDENT_PROGRESSION: '../../data/proxy/${date}/incident-progression-${hash}.json',
- API_TIMELINE: '../../data/proxy/${date}/timeline-${hash}.tsv'
-};
-
-module.exports = ProxyConstants;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/notebooks.js b/spot-oa/ui/proxy/js/notebooks.js
new file mode 100644
index 0000000..1ee6e66
--- /dev/null
+++ b/spot-oa/ui/proxy/js/notebooks.js
@@ -0,0 +1,72 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+ (
+ <form className="form-inline">
+ <div className="form-group">
+ <label htmlFor="dataDatePicker">Data Date:</label>
+ <div className="input-group input-group-xs">
+ <DateInput id="dataDatePicker" />
+ <div className="input-group-addon">
+ <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+ </div>
+ </div>
+ </div>
+ </form>
+ ),
+ document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+ <div id="spot-content">
+ <PanelRow maximized>
+ <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+ <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="proxy/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+ </Panel>
+ </PanelRow>
+ </div>,
+ document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/CommentsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/CommentsStore.js b/spot-oa/ui/proxy/js/stores/CommentsStore.js
index a31456c..eefc400 100755
--- a/spot-oa/ui/proxy/js/stores/CommentsStore.js
+++ b/spot-oa/ui/proxy/js/stores/CommentsStore.js
@@ -1,33 +1,64 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-var d3 = require('d3');
-
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var ProxyConstants = require('../constants/ProxyConstants');
-var RestStore = require('../../../js/stores/RestStore');
-
-var CommentsStore = assign(new RestStore(ProxyConstants.API_COMMENTS), {
- _parser: d3.dsv('|', 'text/plain'),
- errorMessages: {
- 404: 'Please choose a different date, no comments have been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(ProxyConstants.API_COMMENTS.replace('${date}', date.replace(/-/g, '')));
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+
+class CommentsStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!) {
+ proxy {
+ threats {
+ comments(date:$date) {
+ uri
+ title
+ summary: text
+ }
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.proxy.threats.comments;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+}
+
+const cs = new CommentsStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- CommentsStore.setDate(action.date);
+ cs.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- CommentsStore.reload();
+ cs.sendQuery();
break;
}
});
-module.exports = CommentsStore;
+module.exports = cs;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/DetailsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/DetailsStore.js b/spot-oa/ui/proxy/js/stores/DetailsStore.js
index a3c2a4d..33d38b4 100755
--- a/spot-oa/ui/proxy/js/stores/DetailsStore.js
+++ b/spot-oa/ui/proxy/js/stores/DetailsStore.js
@@ -1,64 +1,114 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
-var assign = require('object-assign');
-var d3 = require('d3');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var ProxyConstants = require('../constants/ProxyConstants');
-var RestStore = require('../../../js/stores/RestStore');
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
-var CLIENT_IP_FILTER = 'clientip';
-var HASH_FILTER = 'hash';
+const DATE_VAR = 'date';
+const URI_VAR = 'uri';
+const CLIENT_IP_VAR = 'clientIp';
-var DetailsStore = assign(new RestStore(ProxyConstants.API_DETAILS), {
- _parser: d3.dsv('\t', 'text/plain'),
- errorMessages: {
- 404: 'No details available'
- },
- headers: {
- p_date: 'Time',
- clientip: 'Client IP',
- host: 'Host',
- webcat: 'Web Category',
- respcode_name: 'Response Code',
- reqmethod: 'Request Method',
- useragent: 'User Agent',
- resconttype: 'MIME Type',
- referer: 'Referer',
- uriport: 'URI Port',
- serverip: 'Proxy IP',
- scbytes: 'Server Bytes',
- csbytes: 'Client Bytes',
- fulluri: 'Full URI'
- },
- setDate: function (date) {
- this.setEndpoint(ProxyConstants.API_DETAILS.replace('${date}', date.replace(/-/g, '')));
- },
- setClientIp: function (clientIp) {
- this.setRestFilter(CLIENT_IP_FILTER, clientIp);
- },
- setHash: function (hash) {
- this.setRestFilter(HASH_FILTER, hash);
+class DetailsStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+
+ this.headers = {
+ datetime: 'Time',
+ clientip: 'Client IP',
+ host: 'Host',
+ webcat: 'Web Category',
+ respcode_name: 'Response Code',
+ reqmethod: 'Request Method',
+ useragent: 'User Agent',
+ resconttype: 'MIME Type',
+ referer: 'Referer',
+ uriport: 'URI Port',
+ serverip: 'Proxy IP',
+ scbytes: 'Server Bytes',
+ csbytes: 'Client Bytes',
+ fulluri: 'Full URI'
+ };
+
+ this.ITERATOR = ['datetime', 'clientip', 'host', 'webcat', 'respcode_name', 'reqmethod', 'useragent', 'resconttype', 'referer', 'uriport', 'serverip', 'scbytes', 'csbytes', 'fulluri'];
}
-});
+
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$uri:String!,$clientIp:SpotIpType!) {
+ proxy {
+ edgeDetails(date:$date,uri:$uri,clientIp:$clientIp) {
+ uriport: uriPort
+ webcat: webCategory
+ resconttype: responseContentType
+ datetime
+ host
+ referer
+ csbytes: clientToServerBytes
+ useragent: userAgent
+ fulluri: uri
+ serverip: serverIp
+ reqmethod: requestMethod
+ respcode: responseCode
+ respcode_name: responseCodeLabel
+ clientip: clientIp
+ scbytes: serverToClientBytes
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.proxy.edgeDetails;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setClientIp(clientIp) {
+ this.setVariable(CLIENT_IP_VAR, clientIp);
+ }
+
+ setUri(uri) {
+ this.setVariable(URI_VAR, uri);
+ }
+}
+
+const ds = new DetailsStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- DetailsStore.setDate(action.date);
+ ds.setDate(action.date);
break;
case SpotConstants.SELECT_THREAT:
- DetailsStore.setClientIp(action.threat[CLIENT_IP_FILTER]);
- DetailsStore.setHash(action.threat[HASH_FILTER].replace(/\//g, '-'));
+ ds.setClientIp(action.threat.clientip);
+ ds.setUri(action.threat.fulluri);
break;
case SpotConstants.RELOAD_SUSPICIOUS:
- DetailsStore.resetData();
+ ds.resetData();
break;
case SpotConstants.RELOAD_DETAILS:
- DetailsStore.reload();
+ ds.sendQuery();
break;
}
});
-module.exports = DetailsStore;
+module.exports = ds;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/IncidentProgressionStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/IncidentProgressionStore.js b/spot-oa/ui/proxy/js/stores/IncidentProgressionStore.js
index a35bb70..961e00f 100755
--- a/spot-oa/ui/proxy/js/stores/IncidentProgressionStore.js
+++ b/spot-oa/ui/proxy/js/stores/IncidentProgressionStore.js
@@ -1,41 +1,80 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
-var assign = require('object-assign');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var JsonStore = require('../../../js/stores/JsonStore');
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
-var ProxyConstants = require('../constants/ProxyConstants');
+const DATE_VAR = 'date';
+const URI_VAR = 'uri';
-var IncidentProgressionStore = assign(new JsonStore(ProxyConstants.API_INCIDENT_PROGRESSION), {
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date) {
- this.setEndpoint(ProxyConstants.API_INCIDENT_PROGRESSION.replace('${date}', date.replace(/-/g, '')));
- },
- setHash: function (hash) {
- this.setRestFilter('hash', hash);
+class IncidentProgressionStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$uri:String!) {
+ proxy {
+ threat {
+ incidentProgression(date:$date,uri:$uri) {
+ requests {
+ reqmethod: requestMethod
+ clientip: clientIp
+ resconttype: responseContentType
+ referer
+ }
+ fulluri: uri
+ referer_for: refererFor
+ }
+ }
+ }
+ }
+ `;
}
-});
+
+ unboxData(data) {
+ return data.proxy.threat.incidentProgression;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setUri(uri) {
+ this.setVariable(URI_VAR, uri);
+ }
+}
+
+const ips = new IncidentProgressionStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- IncidentProgressionStore.setDate(action.date);
+ ips.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- IncidentProgressionStore.removeRestFilter('hash');
- IncidentProgressionStore.resetData();
+ ips.resetData();
break;
case SpotConstants.SELECT_COMMENT:
- IncidentProgressionStore.setHash(action.comment.hash);
- IncidentProgressionStore.reload();
+ ips.setUri(action.comment.uri);
+ ips.sendQuery();
break;
}
});
-module.exports = IncidentProgressionStore;
+module.exports = ips;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/NotebookStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/NotebookStore.js b/spot-oa/ui/proxy/js/stores/NotebookStore.js
new file mode 100644
index 0000000..31cffc6
--- /dev/null
+++ b/spot-oa/ui/proxy/js/stores/NotebookStore.js
@@ -0,0 +1,107 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SuspiciousStore = require('./SuspiciousStore');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const SCORED_ELEMENTS = 'input';
+const RESET_SCORED_ELEMENTS = 'date';
+
+class NotebookStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+ this.selectedQuery = 'scoreQuery';
+ this.query = {
+ scoreQuery: `
+ mutation($input:[ProxyScoreInputType!]!) {
+ proxy{
+ score(input:$input)
+ {success}
+ }
+ }`,
+ resetQuery: `
+ mutation($date:SpotDateType!) {
+ proxy{
+ resetScoredConnections(date:$date){
+ success
+ }
+ }
+ }`
+ };
+ this.completeClass = false;
+ }
+
+ getQuery() {
+ return this.query[this.selectedQuery];
+ }
+
+ unboxData(data) {
+ return data.flow.suspicious;
+ }
+
+ setScoredElements(scoredElements) {
+ this.selectedQuery = 'scoreQuery';
+ this.setVariable(SCORED_ELEMENTS, scoredElements);
+ this.unsetVariables(RESET_SCORED_ELEMENTS);
+ }
+
+ resetScoredElements(date) {
+ this.selectedQuery = 'resetQuery';
+ this.setVariable(RESET_SCORED_ELEMENTS, date);
+ this.unsetVariables(SCORED_ELEMENTS);
+ }
+
+ unsetVariables(variable) {
+ this.unsetVariable(variable);
+ }
+
+ reloadElements() {
+ SuspiciousStore.sendQuery();
+ }
+
+ changeCssCls() {
+ this.completeClass = !this.completeClass;
+ }
+
+
+}
+
+const ns = new NotebookStore();
+
+SpotDispatcher.register(function (action) {
+ switch (action.actionType) {
+ case SpotConstants.SAVE_SCORED_ELEMENTS:
+ ns.setScoredElements(action.scoredElems);
+ ns.sendQuery();
+ break;
+ case SpotConstants.RESET_SCORED_ELEMENTS:
+ ns.resetScoredElements(action.date);
+ ns.sendQuery();
+ break;
+ case SpotConstants.RELOAD_SUSPICIOUS:
+ ns.resetData();
+ break;
+ case SpotConstants.CHANGE_CSS_CLS:
+ ns.changeCssCls();
+ break;
+ }
+});
+
+module.exports = ns;
[10/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/README.md b/spot-oa/oa/dns/README.md
index aab8673..01d2327 100644
--- a/spot-oa/oa/dns/README.md
+++ b/spot-oa/oa/dns/README.md
@@ -1,10 +1,10 @@
# DNS
-DNS sub-module extracts and transforms DNS (Domain Name Service) data already ranked by spot-ml and will load into csv files for presentation layer.
+DNS sub-module extracts and transforms DNS (Domain Name Service) data already ranked by spot-ml and will load it into impala tables for the presentation layer.
## DNS Components
-###dns_oa.py
+### dns_oa.py
DNS spot-oa main script executes the following steps:
@@ -15,30 +15,26 @@ DNS spot-oa main script executes the following steps:
ipython Notebooks: ipynb/dns/<date>/
2. Creates a copy of the notebooks templates into the ipython Notebooks path and renames them removing the "_master" part from the name.
-
+
3. Gets the dns_results.csv from the HDFS location according to the selected date, and copies it back to the corresponding data path.
-
+
4. Reads a given number of rows from the results file.
- 5. Gets the top level domain out of the dns_qry_name, and adds it in the new column 'tld'
-
+ 5. Gets the top level domain out of the dns_qry_name, and adds it in the new column 'tld'.
+
6. Checks reputation for the query_name of each connection.
-
+
7. Adds two new columns for the severity of the query_name and the client ip of each connection.
8. Adds a new column with the hour part of the frame_time.
-
- 9. Translates the 'dns_query_class', 'dns_query_type','dns_query_rcode' to human readable text according to the IANA specification. The translated values are stored in the dns_qry_class_name, dns_qry_type_name, dns_qry_rcode_name columns, respectively.
-
+
+ 9. Translates the 'dns_query_class', 'dns_query_type','dns_query_rcode' to human readable text according to the IANA specification. The translated values are stored in the dns_qry_class_name, dns_qry_type_name, dns_qry_rcode_name columns, respectively.
+
10. Adds Network Context.
-
- 11. Saves dns_scores.csv file.
-
- 12. Creates a backup of dns_scores.csv file named dns_scores_bu.csv.
-
- 13. Creates dns data details files.
-
- 14. Creates dendrogram data files.
+
+ 11. Saves results to the dns_scores table.
+
+ 12. Generates details and dendrogram diagram data. These details include information about aditional connections to display the details table in the UI.
**Dependencies**
@@ -51,9 +47,8 @@ DNS spot-oa main script executes the following steps:
- [components/data](/spot-oa/oa/components#data)
- [components/nc](/spot-oa/oa/components#network-context-nc)
- [components/reputation](/spot-oa/oa/components/reputation)
-- dns_conf.json
-
-
+- dns_conf.json
+
**Prerequisites**
@@ -70,12 +65,12 @@ Before running DNS OA users need to configure components for the first time. It
**Output**
-- dns_scores.csv: Main results file for DNS OA. This file will contain suspicious connects information and it's limited to the number of rows the user selected when running [oa/start_oa.py](/spot-oa/oa/INSTALL.md#usage).
-
- Schema with zero-indexed columns:
-
+- DNS suspicious connections. _dns\_scores_ table.
+
+Main results for Flow OA. Main results file for DNS OA. The data stored in this table is limited by the number of rows the user selected when running [oa/start_oa.py](/spot-oa/oa/INSTALL.md#usage).
+
0.frame_time: string
- 1.frame_len: int
+ 1.unix_tstamp: bigint
2.ip_dst: string
3.dns_qry_name: string
4.dns_qry_class: string
@@ -84,49 +79,60 @@ Before running DNS OA users need to configure components for the first time. It
7.score: double
8.tld: string
9.query_rep: string
- 10.hh: string
- 11.ip_sev: int
- 12.dns_sev: int
- 13.dns_qry_class_name: string
- 14.dns_qry_type_name: string
- 15.dns_qry_rcode_name: string
- 16.network_context: string
- 17.unix_tstamp: bigint
+ 10.hh: string
+ 11.dns_qry_class_name: string
+ 12.dns_qry_type_name: string
+ 13.dns_qry_rcode_name: string
+ 14.network_context: string
-- dns_scores_bu.csv: The backup file of suspicious connects in case user wants to roll back any changes made during analysis. Schema is same as dns_scores.csv.
+- DNS details _dns\_scores_ table.
-- dendro-\<DNS query name>.csv: One file for each source IP. This file includes information about all the queries made to a particular DNS query name. The number of retrieved rows is limited by the value of "\_details\_limit" parameter
+One file for each source IP. This file includes information about all the queries made to a particular DNS query name. The number of retrieved rows is limited by the value of "\_details\_limit" parameter
+
+ 0.unix_tstamp bigint
+ 1.dns_a string
+ 2.dns_qry_name string
+ 3.ip_dst string
- Schema with zero-indexed columns:
-
- 0.dns_a: string
- 1.dns_qry_name: string
- 2.ip_dst: string
-- edge-\<DNS query name>_\<HH>_00.csv: One file for each DNS query name for each hour of the day. This file contains details for each
-connection between DNS and source IP.
-
- Schema with zero-indexed columns:
-
- 0.frame_time: string
- 1.frame_len: int
- 2.ip_dst: string
- 3.ip_src: string
- 4.dns_qry_name: string
- 5.dns_qry_class_name: string
- 6.dns_qry_type_name: string
- 7.dns_qry_rcode_name: string
- 8.dns_a: string
+- DNS Details: _dns\_dendro_ table.
+One file for each DNS query name for each hour of the day. This file contains details for each
+connection between DNS and source IP.
+
+ 0.unix_tstamp bigint
+ 1.frame_len bigint
+ 2.ip_dst string
+ 3.ip_src string
+ 4.ns_qry_name string
+ 5.dns_qry_class string
+ 6.dns_qry_type int
+ 7.dns_qry_rcode int
+ 8.dns_a string
+ 9.hh int
+ 10.dns_qry_class_name string
+ 11.dns_qry_type_name string
+ 12.dns_qry_rcode_name string
+ 13.network_context string
+
+
+- DNS Ingest summary. _dns\_ingest\_summary_ table.
+
+This table is populated with the number of connections ingested by minute during that day.
+
+ Table schema:
+ 0. tdate: string
+ 1. total: bigint
+
-###dns_conf.json
-This file is part of the initial configuration for the DNS pipeline. It will contain mapped all the columns included in the dns_results.csv and dns_scores.csv files.
+### dns_conf.json
+This file is part of the initial configuration for the DNS pipeline. It will contain mapped all the columns included in the _dns\_edge_ and _dns\_dendro_ tables.
This file contains three main arrays:
- dns_results_fields: Reference of the column name and indexes in the dns_results.csv file.
- - dns_score_fields: Reference of the column name and indexes in the dns_scores.csv file.
+ - dns_score_fields: Reference of the column name and indexes in the _dns\_edge_ table.
- add_reputation: According to the dns_results.csv file, this is the column index of the value which will be evaluated using the reputation services.
@@ -136,6 +142,9 @@ After OA process completes, a copy of each iPython notebook is going to be copie
With these iPython notebooks user will be able to perform further analysis and score connections. User can also
experiment adding or modifying the code.
If a new functionality is required for the ipython notebook, the templates need to be modified to include the functionality for new executions.
-For further reference on how to work with these notebooks, you can read:
-[Edge Notebook.ipynb](/spot-oa/oa/dns/ipynb_templates/EdgeNotebook.md)
-[Threat_Investigation.ipynb](/spot-oa/oa/dns/ipynb_templates/ThreatInvestigation.md)
\ No newline at end of file
+For further reference on how to work with these notebooks, you can read:
+[Threat_Investigation.ipynb](/spot-oa/oa/dns/ipynb_templates/ThreatInvestigation.md)
+
+
+### Reset scored connections
+To reset all scored connections for a day, a specific cell with a preloaded function is included in the Advanced Mode Notebook. The cell is commented to avoid accidental executions, but is properly labeled.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/__init__.py b/spot-oa/oa/dns/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/dns/__init__.py
+++ b/spot-oa/oa/dns/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/dns_conf.json
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/dns_conf.json b/spot-oa/oa/dns/dns_conf.json
index aadd92c..0e23a38 100644
--- a/spot-oa/oa/dns/dns_conf.json
+++ b/spot-oa/oa/dns/dns_conf.json
@@ -11,27 +11,25 @@
, "score" : 8
},
"dns_score_fields":
- {
+ {
"frame_time" : 0
- , "frame_len" : 1
- , "ip_dst": 2
- , "dns_qry_name" : 3
- , "dns_qry_class" : 4
- , "dns_qry_type" : 5
- , "dns_qry_rcode" : 6
- , "score" : 7
- , "tld" : 8
- , "query_rep" : 9
- , "hh": 10
- , "ip_sev" : 11
- , "dns_sev" : 12
- , "dns_qry_class_name" : 13
- , "dns_qry_type_name" : 14
- , "dns_qry_rcode_name" : 15
- , "network_context" : 16
- , "unix_tstamp": 17
+ ,"unix_tstamp" : 1
+ , "frame_len" : 2
+ , "ip_dst": 3
+ , "dns_qry_name" : 4
+ , "dns_qry_class" : 5
+ , "dns_qry_type" : 6
+ , "dns_qry_rcode" : 7
+ , "score" : 8
+ , "tld" : 9
+ , "query_rep" : 10
+ , "hh": 11
+ , "dns_qry_class_name" : 12
+ , "dns_qry_type_name" : 13
+ , "dns_qry_rcode_name" : 14
+ , "network_context" : 15
},
"add_reputation":{
"query_rep":4
}
-}
\ No newline at end of file
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/dns_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/dns_oa.py b/spot-oa/oa/dns/dns_oa.py
index 2033c89..5982e8b 100644
--- a/spot-oa/oa/dns/dns_oa.py
+++ b/spot-oa/oa/dns/dns_oa.py
@@ -23,7 +23,8 @@ import sys
import datetime
import csv, math
from tld import get_tld
-
+import api.resources.impala_engine as impala
+import api.resources.hdfs_client as HDFSClient
from collections import OrderedDict
from utils import Util
from components.data.data import Data
@@ -67,7 +68,6 @@ class OA(object):
# initialize data engine
self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '')
- self._engine = Data(self._db,self._table_name ,self._logger)
def start(self):
@@ -76,15 +76,16 @@ class OA(object):
start = time.time()
####################
+ self._clear_previous_executions()
self._create_folder_structure()
self._add_ipynb()
self._get_dns_results()
self._add_tld_column()
self._add_reputation()
- self._add_hh_and_severity()
+ self._add_hh_column()
self._add_iana()
self._add_network_context()
- self._create_dns_scores_csv()
+ self._create_dns_scores()
self._get_oa_details()
self._ingest_summary()
@@ -93,18 +94,40 @@ class OA(object):
print(end - start)
##################
+
+ def _clear_previous_executions(self):
+
+ self._logger.info("Cleaning data from previous executions for the day")
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ table_schema = []
+ HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
+ table_schema=['suspicious', 'edge', 'dendro', 'threat_dendro', 'threat_investigation', 'storyboard', 'summary' ]
+
+ for path in table_schema:
+ HDFSClient.delete_folder("{0}/{1}/hive/oa/{2}/y={3}/m={4}/d={5}".format(HUSER,self._table_name,path,yr,int(mn),int(dy)),user="impala")
+ impala.execute_query("invalidate metadata")
+
+ #removes Feedback file
+ HDFSClient.delete_folder("{0}/{1}/scored_results/{2}{3}{4}/feedback/ml_feedback.csv".format(HUSER,self._table_name,yr,mn,dy))
+ #removes json files from the storyboard
+ HDFSClient.delete_folder("{0}/{1}/oa/{2}/{3}/{4}/{5}".format(HUSER,self._table_name,"storyboard",yr,mn,dy))
+
+
def _create_folder_structure(self):
# create date folder structure if it does not exist.
self._logger.info("Creating folder structure for OA (data and ipynb)")
self._data_path,self._ingest_summary_path,self._ipynb_path = Util.create_oa_folders("dns",self._date)
+
def _add_ipynb(self):
if os.path.isdir(self._ipynb_path):
- self._logger.info("Adding edge investigation IPython Notebook")
- shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+ self._logger.info("Adding advanced mode IPython Notebook")
+ shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
self._logger.info("Adding threat investigation IPython Notebook")
shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
@@ -126,58 +149,59 @@ class OA(object):
get_command = Util.get_ml_results_form_hdfs(hdfs_path,self._data_path)
self._logger.info("{0}".format(get_command))
- # validate files exists
if os.path.isfile(dns_results):
# read number of results based in the limit specified.
self._logger.info("Reading {0} dns results file: {1}".format(self._date,dns_results))
self._dns_results = Util.read_results(dns_results,self._limit,self._results_delimiter)[:]
- if len(self._dns_results) == 0: self._logger.error("There are not flow results.");sys.exit(1)
+ if len(self._dns_results) == 0: self._logger.error("There are not dns results.");sys.exit(1)
else:
self._logger.error("There was an error getting ML results from HDFS")
sys.exit(1)
- # add headers.
- self._logger.info("Adding headers")
- self._dns_scores_headers = [ str(key) for (key,value) in self._conf['dns_score_fields'].items() ]
-
# add dns content.
- self._dns_scores = [ conn[:] for conn in self._dns_results][:]
+ self._dns_scores = [ conn[:] for conn in self._dns_results][:]
+
def _move_time_stamp(self,dns_data):
- for dns in dns_data:
- time_stamp = dns[1]
- dns.remove(time_stamp)
- dns.append(time_stamp)
-
+ # return dns_data_ordered
return dns_data
- def _create_dns_scores_csv(self):
+
+ def _create_dns_scores(self):
- dns_scores_csv = "{0}/dns_scores.csv".format(self._data_path)
- dns_scores_final = self._move_time_stamp(self._dns_scores)
- dns_scores_final.insert(0,self._dns_scores_headers)
- Util.create_csv_file(dns_scores_csv,dns_scores_final)
+ # get date parameters.
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ value_string = ""
+
+ dns_scores_final = self._move_time_stamp(self._dns_scores)
+ self._dns_scores = dns_scores_final
- # create bk file
- dns_scores_bu_csv = "{0}/dns_scores_bu.csv".format(self._data_path)
- Util.create_csv_file(dns_scores_bu_csv,dns_scores_final)
+ for row in dns_scores_final:
+ value_string += str(tuple(Util.cast_val(item) for item in row)) + ","
+
+ load_into_impala = ("""
+ INSERT INTO {0}.dns_scores partition(y={2}, m={3}, d={4}) VALUES {1}
+ """).format(self._db, value_string[:-1], yr, mn, dy)
+ impala.execute_query(load_into_impala)
def _add_tld_column(self):
qry_name_col = self._conf['dns_results_fields']['dns_qry_name']
- self._dns_scores = [conn + [ get_tld("http://" + str(conn[qry_name_col]), fail_silently=True) if "http://" not in str(conn[qry_name_col]) else get_tld(str(conn[qry_name_col]), fail_silently=True)] for conn in self._dns_scores ]
-
+ self._dns_scores = [conn + [ get_tld("http://" + str(conn[qry_name_col]), fail_silently=True) if "http://" not in str(conn[qry_name_col]) else get_tld(str(conn[qry_name_col]), fail_silently=True)] for conn in self._dns_scores ]
+
+
def _add_reputation(self):
# read configuration.
reputation_conf_file = "{0}/components/reputation/reputation_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
self._logger.info("Reading reputation configuration file: {0}".format(reputation_conf_file))
rep_conf = json.loads(open(reputation_conf_file).read())
-
-
+
# initialize reputation services.
self._rep_services = []
self._logger.info("Initializing reputation services.")
@@ -199,7 +223,6 @@ class OA(object):
# get reputation per column.
self._logger.info("Getting reputation for each service in config")
rep_services_results = []
-
if self._rep_services :
for key,value in rep_cols.items():
@@ -208,21 +231,22 @@ class OA(object):
for result in rep_services_results:
rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
- self._dns_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._dns_scores ]
+ if rep_results:
+ self._dns_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._dns_scores ]
+ else:
+ self._dns_scores = [ conn + [""] for conn in self._dns_scores ]
else:
self._dns_scores = [ conn + [""] for conn in self._dns_scores ]
+ def _add_hh_column(self):
- def _add_hh_and_severity(self):
-
- # add hh value and sev columns.
+ # add hh value column.
dns_date_index = self._conf["dns_results_fields"]["frame_time"]
- self._dns_scores = [conn + [ filter(None,conn[dns_date_index].split(" "))[3].split(":")[0]] + [0] + [0] for conn in self._dns_scores ]
+ self._dns_scores = [conn + [ filter(None,conn[dns_date_index].split(" "))[3].split(":")[0]] for conn in self._dns_scores ]
def _add_iana(self):
-
iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if os.path.isfile(iana_conf_file):
iana_config = json.loads(open(iana_conf_file).read())
@@ -231,13 +255,13 @@ class OA(object):
dns_qry_class_index = self._conf["dns_results_fields"]["dns_qry_class"]
dns_qry_type_index = self._conf["dns_results_fields"]["dns_qry_type"]
dns_qry_rcode_index = self._conf["dns_results_fields"]["dns_qry_rcode"]
- self._dns_scores = [ conn + [ dns_iana.get_name(conn[dns_qry_class_index],"dns_qry_class")] + [dns_iana.get_name(conn[dns_qry_type_index],"dns_qry_type")] + [ dns_iana.get_name(conn[dns_qry_rcode_index],"dns_qry_rcode") ] for conn in self._dns_scores ]
+ self._dns_scores = [ conn + [ dns_iana.get_name(conn[dns_qry_class_index],"dns_qry_class")] + [dns_iana.get_name(conn[dns_qry_type_index],"dns_qry_type")] + [dns_iana.get_name(conn[dns_qry_rcode_index],"dns_qry_rcode")] for conn in self._dns_scores ]
else:
self._dns_scores = [ conn + ["","",""] for conn in self._dns_scores ]
- def _add_network_context(self):
+ def _add_network_context(self):
nc_conf_file = "{0}/components/nc/nc_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if os.path.isfile(nc_conf_file):
nc_conf = json.loads(open(nc_conf_file).read())["NC"]
@@ -245,15 +269,15 @@ class OA(object):
ip_dst_index = self._conf["dns_results_fields"]["ip_dst"]
self._dns_scores = [ conn + [dns_nc.get_nc(conn[ip_dst_index])] for conn in self._dns_scores ]
else:
- self._dns_scores = [ conn + [""] for conn in self._dns_scores ]
+ self._dns_scores = [ conn + [0] for conn in self._dns_scores ]
def _get_oa_details(self):
- self._logger.info("Getting OA DNS suspicious details/chord diagram")
+ self._logger.info("Getting OA DNS suspicious details/dendro diagram")
# start suspicious connects details process.
p_sp = Process(target=self._get_suspicious_details)
- p_sp.start()
+ p_sp.start()
# start chord diagram process.
p_dn = Process(target=self._get_dns_dendrogram)
@@ -262,6 +286,7 @@ class OA(object):
p_sp.join()
p_dn.join()
+
def _get_suspicious_details(self):
iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -270,85 +295,87 @@ class OA(object):
dns_iana = IanaTransform(iana_config["IANA"])
for conn in self._dns_scores:
- # get data to query
- date=conn[self._conf["dns_score_fields"]["frame_time"]].split(" ")
- date = filter(None,date)
-
- if len(date) == 5:
- year=date[2]
- month=datetime.datetime.strptime(date[0], '%b').strftime('%m')
- day=date[1]
- hh=conn[self._conf["dns_score_fields"]["hh"]]
- dns_qry_name = conn[self._conf["dns_score_fields"]["dns_qry_name"]]
- self._get_dns_details(dns_qry_name,year,month,day,hh,dns_iana)
+
+ timestamp = conn[self._conf["dns_score_fields"]["unix_tstamp"]]
+ full_date = datetime.datetime.utcfromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
+
+ date = full_date.split(" ")[0].split("-")
+ # get date parameters.
+ yr = date[0]
+ mn = date[1]
+ dy = date[2]
+ time = full_date.split(" ")[1].split(":")
+ hh = int(time[0])
+
+ dns_qry_name = conn[self._conf["dns_score_fields"]["dns_qry_name"]]
+ self._get_dns_details(dns_qry_name,yr,mn,dy,hh,dns_iana)
+
def _get_dns_details(self,dns_qry_name,year,month,day,hh,dns_iana):
-
- limit = self._details_limit
- edge_file ="{0}/edge-{1}_{2}_00.csv".format(self._data_path,dns_qry_name.replace("/","-"),hh)
- edge_tmp ="{0}/edge-{1}_{2}_00.tmp".format(self._data_path,dns_qry_name.replace("/","-"),hh)
+ value_string = ""
+ query_to_load =("""
+ SELECT unix_tstamp,frame_len,ip_dst,ip_src,dns_qry_name,dns_qry_class,dns_qry_type,dns_qry_rcode,dns_a,h as hh
+ FROM {0}.{1} WHERE y={2} AND m={3} AND d={4} AND dns_qry_name LIKE '%{5}%' AND h={6} LIMIT {7};
+ """).format(self._db,self._table_name,year,month,day,dns_qry_name,hh,self._details_limit)
+
+ try:
+ dns_details = impala.execute_query(query_to_load)
+ except:
+ self._logger.info("WARNING. Details couldn't be retreived for {0}, skipping this step".format(dns_qry_name))
+ else:
+ # add IANA to results.
+ update_rows = []
+ if dns_iana:
+ self._logger.info("Adding IANA translation to details results")
- if not os.path.isfile(edge_file):
-
- dns_qry = ("SELECT frame_time,frame_len,ip_dst,ip_src,dns_qry_name,dns_qry_class,dns_qry_type,dns_qry_rcode,dns_a FROM {0}.{1} WHERE y={2} AND m={3} AND d={4} AND dns_qry_name LIKE '%{5}%' AND h={6} LIMIT {7};").format(self._db,self._table_name,year,month,day,dns_qry_name,hh,limit)
-
- # execute query
- try:
- self._engine.query(dns_qry,edge_tmp)
- except:
- self._logger.error("ERROR. Edge file couldn't be created for {0}, skipping this step".format(dns_qry_name))
+ dns_details = [ conn + (dns_iana.get_name(str(conn[5]),"dns_qry_class"),dns_iana.get_name(str(conn[6]),"dns_qry_type"),dns_iana.get_name(str(conn[7]),"dns_qry_rcode")) for conn in dns_details ]
+ else:
+ self._logger.info("WARNING: NO IANA configured.")
+ dns_details = [ conn + ("","","") for conn in dns_details ]
+
+ nc_conf_file = "{0}/components/nc/nc_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ if os.path.isfile(nc_conf_file):
+ nc_conf = json.loads(open(nc_conf_file).read())["NC"]
+ dns_nc = NetworkContext(nc_conf,self._logger)
+ dns_details = [ conn + (dns_nc.get_nc(conn[2]),) for conn in dns_details ]
else:
- # add IANA to results.
- if dns_iana:
- update_rows = []
- self._logger.info("Adding IANA translation to details results")
- with open(edge_tmp) as dns_details_csv:
- rows = csv.reader(dns_details_csv, delimiter=',', quotechar='|')
- try:
- next(rows)
- update_rows = [[conn[0]] + [conn[1]] + [conn[2]] + [conn[3]] + [conn[4]] + [dns_iana.get_name(conn[5],"dns_qry_class")] + [dns_iana.get_name(conn[6],"dns_qry_type")] + [dns_iana.get_name(conn[7],"dns_qry_rcode")] + [conn[8]] for conn in rows]
- update_rows = filter(None, update_rows)
- header = [ "frame_time", "frame_len", "ip_dst","ip_src","dns_qry_name","dns_qry_class_name","dns_qry_type_name","dns_qry_rcode_name","dns_a" ]
- update_rows.insert(0,header)
- except IndexError:
- pass
+ dns_details = [ conn + (0,) for conn in dns_details ]
- else:
- self._logger.info("WARNING: NO IANA configured.")
-
- # create edge file.
- self._logger.info("Creating edge file:{0}".format(edge_file))
- with open(edge_file,'wb') as dns_details_edge:
- writer = csv.writer(dns_details_edge, quoting=csv.QUOTE_ALL)
- if update_rows:
- writer.writerows(update_rows)
- else:
- shutil.copy(edge_tmp,edge_file)
+ for row in dns_details:
+ value_string += str(tuple(item for item in row)) + ","
+
+ if value_string != "":
- os.remove(edge_tmp)
+ query_to_insert=("""
+ INSERT INTO {0}.dns_edge PARTITION (y={1}, m={2}, d={3}) VALUES ({4});
+ """).format(self._db,year, month, day, value_string[:-1])
+
+ impala.execute_query(query_to_insert)
def _get_dns_dendrogram(self):
- limit = self._details_limit
- for conn in self._dns_scores:
- date=conn[self._conf["dns_score_fields"]["frame_time"]].split(" ")
- date = filter(None,date)
- if len(date) == 5:
- year=date[2]
- month=datetime.datetime.strptime(date[0], '%b').strftime('%m')
- day=date[1]
- ip_dst=conn[self._conf["dns_score_fields"]["ip_dst"]]
- self._get_dendro(self._db,self._table_name,ip_dst,year,month,day, limit)
+ for conn in self._dns_scores:
+ timestamp = conn[self._conf["dns_score_fields"]["unix_tstamp"]]
+ full_date = datetime.datetime.utcfromtimestamp(int(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
+
+ date = full_date.split(" ")[0].split("-")
+ # get date parameters.
+ yr = date[0]
+ mn = date[1]
+ dy = date[2]
+ ip_dst=conn[self._conf["dns_score_fields"]["ip_dst"]]
- def _get_dendro(self,db,table,ip_dst,year,month,day,limit):
+ query_to_load = ("""
+ INSERT INTO TABLE {0}.dns_dendro PARTITION (y={2}, m={3},d={4})
+ SELECT unix_tstamp, dns_a, dns_qry_name, ip_dst
+ FROM (SELECT unix_tstamp, susp.ip_dst, susp.dns_qry_name, susp.dns_a
+ FROM {0}.{1} as susp WHERE susp.y={2} AND susp.m={3} AND susp.d={4} AND susp.ip_dst='{5}'
+ LIMIT {6}) AS tmp GROUP BY dns_a, dns_qry_name, ip_dst, unix_tstamp
+ """).format(self._db,self._table_name,yr,mn,dy,ip_dst,self._details_limit)
- dendro_file = "{0}/dendro-{1}.csv".format(self._data_path,ip_dst)
- if not os.path.isfile(dendro_file):
- dndro_qry = ("SELECT dns_a, dns_qry_name, ip_dst FROM (SELECT susp.ip_dst, susp.dns_qry_name, susp.dns_a FROM {0}.{1} as susp WHERE susp.y={2} AND susp.m={3} AND susp.d={4} AND susp.ip_dst='{5}' LIMIT {6}) AS tmp GROUP BY dns_a, dns_qry_name, ip_dst").format(db,table,year,month,day,ip_dst,limit)
- # execute query
- self._engine.query(dndro_qry,dendro_file)
+ impala.execute_query(query_to_load)
def _ingest_summary(self):
@@ -363,48 +390,32 @@ class OA(object):
result_rows = []
df_filtered = pd.DataFrame()
- ingest_summary_file = "{0}/is_{1}{2}.csv".format(self._ingest_summary_path,yr,mn)
- ingest_summary_tmp = "{0}.tmp".format(ingest_summary_file)
-
- if os.path.isfile(ingest_summary_file):
- df = pd.read_csv(ingest_summary_file, delimiter=',')
- #discards previous rows from the same date
- df_filtered = df[df['date'].str.contains("{0}-{1}-{2}".format(yr, mn, dy)) == False]
- else:
- df = pd.DataFrame()
-
- # get ingest summary.
- ingest_summary_qry = ("SELECT frame_time, COUNT(*) as total "
- " FROM {0}.{1}"
- " WHERE y={2} AND m={3} AND d={4} "
- " AND unix_tstamp IS NOT NULL AND frame_time IS NOT NULL"
- " AND frame_len IS NOT NULL AND dns_qry_name IS NOT NULL"
- " AND ip_src IS NOT NULL "
- " AND (dns_qry_class IS NOT NULL AND dns_qry_type IS NOT NULL AND dns_qry_rcode IS NOT NULL ) "
- " GROUP BY frame_time;")
-
- ingest_summary_qry = ingest_summary_qry.format(self._db,self._table_name, yr, mn, dy)
-
- results_file = "{0}/results_{1}.csv".format(self._ingest_summary_path,self._date)
- self._engine.query(ingest_summary_qry,output_file=results_file,delimiter=",")
-
-
- if os.path.isfile(results_file):
- df_results = pd.read_csv(results_file, delimiter=',')
-
- # Forms a new dataframe splitting the minutes from the time column
- df_new = pd.DataFrame([["{0}-{1}-{2} {3}:{4}".format(yr, mn, dy,val['frame_time'].split(" ")[3].split(":")[0].zfill(2),val['frame_time'].split(" ")[3].split(":")[1].zfill(2)), int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df_results.iterrows()],columns = ingest_summary_cols)
-
- #Groups the data by minute
- sf = df_new.groupby(by=['date'])['total'].sum()
-
- df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})
-
- df_final = df_filtered.append(df_per_min, ignore_index=True)
- df_final.to_csv(ingest_summary_tmp,sep=',', index=False)
-
- os.remove(results_file)
- os.rename(ingest_summary_tmp,ingest_summary_file)
- else:
- self._logger.info("No data found for the ingest summary")
-
+ query_to_load = ("""
+ SELECT frame_time, COUNT(*) as total FROM {0}.{1}
+ WHERE y={2} AND m={3} AND d={4} AND unix_tstamp IS NOT NULL
+ AND frame_time IS NOT NULL AND frame_len IS NOT NULL
+ AND dns_qry_name IS NOT NULL AND ip_src IS NOT NULL
+ AND (dns_qry_class IS NOT NULL AND dns_qry_type IS NOT NULL
+ AND dns_qry_rcode IS NOT NULL ) GROUP BY frame_time;
+ """).format(self._db,self._table_name, yr, mn, dy)
+
+ results = impala.execute_query_as_list(query_to_load)
+ df = pd.DataFrame(results)
+
+ # Forms a new dataframe splitting the minutes from the time column
+ df_new = pd.DataFrame([["{0}-{1}-{2} {3}:{4}".format(yr, mn, dy,\
+ val['frame_time'].replace(" "," ").split(" ")[3].split(":")[0].zfill(2),\
+ val['frame_time'].replace(" "," ").split(" ")[3].split(":")[1].zfill(2)),\
+ int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df.iterrows()],columns = ingest_summary_cols)
+
+ #Groups the data by minute
+ sf = df_new.groupby(by=['date'])['total'].sum()
+ df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})
+
+ df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False)
+
+ if len(df_final) > 0:
+ query_to_insert=("""
+ INSERT INTO {0}.dns_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
+ """).format(self._db, yr, mn, dy, tuple(df_final))
+ impala.execute_query(query_to_insert)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb b/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb
new file mode 100644
index 0000000..e12b1ee
--- /dev/null
+++ b/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb
@@ -0,0 +1,292 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Apache Spot's Ipython Advanced Mode\n",
+ "## DNS\n",
+ "\n",
+ "This guide provides examples about how to request data, show data with some cool libraries like pandas and more.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Import Libraries**\n",
+ "\n",
+ "The next cell will import the necessary libraries to execute the functions. Do not remove"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import linecache, bisect\n",
+ "import os\n",
+ "\n",
+ "spath = os.getcwd()\n",
+ "path = spath.split(\"/\")\n",
+ "date = path[len(path)-1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Request Data**\n",
+ "\n",
+ "In order to request data we are using Graphql (a query language for APIs, more info at: http://graphql.org/).\n",
+ "\n",
+ "We provide the function to make a data request, all you need is a query and variables\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "def makeGraphqlRequest(query, variables):\n",
+ " return GraphQLClient.request(query, variables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now that we have a function, we can run a query like this:\n",
+ "\n",
+ "*Note: There's no need to manually set the date for the query, by default the code will read the date from the current path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "suspicious_query = \"\"\"query($date:SpotDateType) {\n",
+ " dns {\n",
+ " suspicious(date:$date)\n",
+ " { clientIp\n",
+ " clientIpSev\n",
+ " dnsQuery\n",
+ " dnsQueryClass\n",
+ " dnsQueryClassLabel\n",
+ " dnsQueryRcode\n",
+ " dnsQueryRcodeLabel\n",
+ " dnsQueryRep\n",
+ " dnsQuerySev\n",
+ " dnsQueryType\n",
+ " dnsQueryTypeLabel\n",
+ " frameLength\n",
+ " frameTime\n",
+ " networkContext\n",
+ " score\n",
+ " tld\n",
+ " unixTimestamp \n",
+ " }\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ "##If you want to use a different date for your query, switch the \n",
+ "##commented/uncommented following lines\n",
+ "\n",
+ "variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# 'date': \"2016-10-08\"\n",
+ " }\n",
+ " \n",
+ "suspicious_request = makeGraphqlRequest(suspicious_query,variables)\n",
+ "\n",
+ "##The variable suspicious_request will contain the resulting data from the query.\n",
+ "results = suspicious_request['data']['dns']['suspicious']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Pandas Dataframes\n",
+ "\n",
+ "The following cell loads the results into a pandas dataframe\n",
+ "\n",
+ "For more information on how to use pandas, you can learn more here: https://pandas.pydata.org/pandas-docs/stable/10min.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "df = pd.read_json(json.dumps(results))\n",
+ "##Printing only the selected column list from the dataframe\n",
+ "##Unless specified otherwise, \n",
+ "print df[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Additional operations \n",
+ "\n",
+ "Additional operations can be performed on the dataframe like sorting the data, filtering it and grouping it\n",
+ "\n",
+ "**Filtering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "##Filter results where the destination port = 3389\n",
+ "##The resulting data will be stored in df2 \n",
+ "\n",
+ "df2 = df[df['tld'].isin(['sjc04-login.dotomi.com'])]\n",
+ "print df2[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Ordering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "srtd = df.sort_values(by=\"tld\")\n",
+ "print srtd[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Grouping the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "## This command will group the results by pairs of source-destination IP\n",
+ "## summarizing all other columns \n",
+ "grpd = df.groupby(['clientIp','tld']).count()\n",
+ "## This will print the resulting dataframe displaying the input and output bytes columnns\n",
+ "print grpd[[\"dnsQuery\"]]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Reset Scored Connections**\n",
+ "\n",
+ "Uncomment and execute the following cell to reset all scored connections for this day"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# reset_scores = \"\"\"mutation($date:SpotDateType!) {\n",
+ "# dns{\n",
+ "# resetScoredConnections(date:$date){\n",
+ "# success\n",
+ "# }\n",
+ "# }\n",
+ "# }\"\"\"\n",
+ "\n",
+ "\n",
+ "# variables={\n",
+ "# 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# }\n",
+ " \n",
+ "# request = makeGraphqlRequest(reset_scores,variables)\n",
+ "\n",
+ "\n",
+ "# print request['data']['dns']['resetScoredConnections']['success']"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Sandbox\n",
+ "\n",
+ "At this point you can perform your own analysis using the previously provided functions as a guide.\n",
+ "\n",
+ "Happy threat hunting!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "#Your code here"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/ipynb_templates/EdgeNotebook.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/ipynb_templates/EdgeNotebook.md b/spot-oa/oa/dns/ipynb_templates/EdgeNotebook.md
deleted file mode 100644
index 6b1579e..0000000
--- a/spot-oa/oa/dns/ipynb_templates/EdgeNotebook.md
+++ /dev/null
@@ -1,76 +0,0 @@
-#DNS Edge Investigation Notebook
-
-###Dependencies
-- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html)
-- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/)
-- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip)
-
-The following python modules will be imported for the notebook to work correctly:
-
- import urllib2
- import json
- import os
- import csv
- import ipywidgets #For jupyter/ipython >= 1.4
- from IPython.html import widgets # For jupyter/ipython < 1.4
- from IPython.display import display, HTML, clear_output, Javascript
- import datetime
- import subprocess
-
-
-###Pre-requisites
-- Execution of the spot-oa process for DNS
-- Correct setup the spot.conf file. [Read more](/wiki/Edit%20Solution%20Configuration)
-- Have a public key authentication between the current UI node and the ML node. [Read more](/wiki/Configure%20User%20Accounts#configure-user-accounts)
-
-
-##Data source
-
-The whole process in this notebook depends entirely on the existence of `dns_scores.csv` file, which is generated at the OA process.
-The data is directly manipulated on the .csv files, so a `dns_scores_bu.csv` is created as a backup to allow the user to restore the original data at any point,
-and this can be performed executing the last cell on the notebook with the following command:
-
- !cp $sconnectbu $sconnect
-
-
-**Input files**
-All these paths should be relative to the main OA path.
-Schema for these files can be found [here](/spot-oa/oa/dns)
-
- data/dns/<date>/dns_scores.csv
- data/dns/<date>/dns_scores_bu.csv
-
-**Temporary Files**
-
- data/dns/<date>/score_tmp.csv
-
-**Output files**
-
- data/dns/<date>/dns_scores.csv (Updated with severity values)
- data/dns/<date>/dns_scores_fb.csv (File with scored connections that will be used for ML feedback)
-
-###Functions
-**Widget configuration**
-This is not a function, but more like global code to set up styles and widgets to format the output of the notebook.
-
-`data_loader():` - This function loads the source file into a csv dictionary reader with all suspicious unscored connections, creating separated lists for
-the 'client_ip' and 'dns_qry_name'.
- Also displays the widgets for the listboxes, textbox, radiobutton list and the 'Score' and 'Save' buttons.
-
-`fill_list(list_control,source):` - This function loads the given dictionary into a listbox and appends an empty item at the top with the value '--Select--' (Just for design sake)
-
-` assign_score(b):` - This function is executed on the onclick event of the ‘Score’ button. The system will first try to get the value from the 'Quick search' textbox ignoring the selections from the listboxes; in case the textbox is empty, it will then
- get the selected values from the 'Client IP' and 'Query' listboxes to then search through the dns_scores.csv file to find matching values.
-A linear search on the file is then performed:
-The value in the 'Quick Scoring' textbox, will be compared against the `dns_qry_name` column. Partial matches will be considered as a positive match and the `dns_sev` column will be updated to the value selected from the radiobutton list.
-The column `ip_dst` will be compared against the 'Client IP' selected value; if a match is found, the `ip_sev` column will be updated to the value selected from the radiobutton list.
-The column `dns_qry_name` will be compared against the 'Query' selected value; if a match is found, the `dns_sev` column will be updated to the value selected from the radiobutton list.
-Every row will be appended to the `dns_scores_tmp.csv` file. This file will replace the original `dns_scores.csv` at the end of the process.
-
-Only the scored rows will also be appended to the `dns_scores_fb.csv` file, which will later be used for the ML feedback.
-
-`save(b):` - This event is triggered by the 'Save' button, and executes javascript functions to refresh the data on all the panels in Suspicious Connects. Since the data source file has been updated, the scored connections will be removed from all
-the panels, since those panels will only display connections where the `dns_sev` value is zero.
-This function also removes the widget panel and reloads it again to update the results, removing the need of a manual refresh, and calls the `ml_feedback():` function.
-
-`ml_feedback():` - A shell script is executed, transferring thru secure copy the _proxy_scores_fb.csv_ file into ML Master node, where the destination path is defined at the spot.conf file.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb b/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb
deleted file mode 100644
index 5573c9a..0000000
--- a/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb
+++ /dev/null
@@ -1,269 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Get Suspicious DNS"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import urllib2\n",
- "import json\n",
- "import os\n",
- "import csv\n",
- "\n",
- "# getting date from the parent path. \n",
- "path = os.getcwd().split(\"/\") \n",
- "date = path[len(path)-1] \n",
- "dsource = path[len(path)-2] \n",
- "dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
- "\n",
- "sconnect = dpath + 'dns_scores.csv'\n",
- "sconnectbu = dpath + 'dns_scores_bu.csv'\n",
- "score_tmp = dpath + 'score_tmp.csv' \n",
- "score_fbk = dpath + 'dns_scores_fb.csv' "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "def apply_css_to_select(select):\n",
- " select._css = (\n",
- " (None, 'height', '90%'),\n",
- " (None, 'width', '90%'),\n",
- " ('select', 'overflow-x', 'auto'),\n",
- " ('select', 'width', '100%'),\n",
- " ('select', 'margin', 0)\n",
- " )\n",
- "\n",
- "try:\n",
- " import ipywidgets as widgets # For jupyter/ipython >= 1.4\n",
- "except ImportError:\n",
- " from IPython.html import widgets\n",
- "from IPython.display import display, HTML, clear_output, Javascript \n",
- "\n",
- "def fill_list(list_control,source):\n",
- " options_list = ['--Select--'] \n",
- " options_list.extend([s for s in source])\n",
- " list_control.options = options_list\n",
- "\n",
- "# client panel\n",
- "client_header = widgets.HTML(value=\"Client IP\")\n",
- "client_select = widgets.Select(height='90%')\n",
- "apply_css_to_select(client_select)\n",
- "\n",
- "client_box = widgets.Box(width='20%', height='100%')\n",
- "client_box.children = [client_header, client_select]\n",
- "\n",
- "# query panel\n",
- "query_header = widgets.HTML(value=\"Query\")\n",
- "query_select = widgets.Select(height='90%')\n",
- "apply_css_to_select(query_select)\n",
- "\n",
- "query_box = widgets.Box(width='60%', height='100%')\n",
- "query_box.children = [query_header, query_select]\n",
- "\n",
- "# Actions Panel\n",
- "actions_header = widgets.HTML(value=\" \")\n",
- "quick_text = widgets.Text(value='', width='100%', placeholder='Quick scoring')\n",
- "quick_text._css = (\n",
- " (None, 'width', '100%'),\n",
- ")\n",
- "rating_btn = widgets.RadioButtons(description='Rating:', options=['1', '2', '3'], width='100%')\n",
- "assign_btn = widgets.Button(description='Score', width='45%')\n",
- "assign_btn.button_style = 'primary'\n",
- "save_btn = widgets.Button(description='Save', width='45%')\n",
- "save_btn.button_style = 'primary'\n",
- "save_btn._css = (\n",
- " (None, 'margin-left', '10%'),\n",
- ")\n",
- "actions_box = widgets.Box(width='20%', height='100%')\n",
- "actions_box.children = [actions_header,quick_text,rating_btn, assign_btn,save_btn]\n",
- "\n",
- "scoring_form = widgets.HBox(width='90%', height=250)\n",
- "scoring_form.children = [client_box,query_box,actions_box]\n",
- "\n",
- "\n",
- "def data_loader(): \n",
- " us_ips = []\n",
- " us_dns = []\n",
- "\n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f, delimiter=',')\n",
- " for row in reader: \n",
- " if row['ip_dst'] not in us_ips and row['ip_sev'] == '0': \n",
- " us_ips.append(row['ip_dst'])\n",
- " if row['dns_qry_name'] not in us_dns and row['dns_sev'] == '0':\n",
- " us_dns.append(row['dns_qry_name']) \n",
- "\n",
- " fill_list(client_select,us_ips)\n",
- " fill_list(query_select,us_dns)\n",
- " client_select.value = \"--Select--\"\n",
- " query_select.value = \"--Select--\" \n",
- "\n",
- "\n",
- "display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- "data_loader()\n",
- "display(scoring_form)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Update Suspicious DNS"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import csv\n",
- "import datetime\n",
- "import subprocess \n",
- "\n",
- "def assign_score(b):\n",
- " score_values = []\n",
- " scored_threats = []\n",
- " ip_sev = int(rating_btn.selected_label) if not \"--Select--\" in client_select.value else \"\"\n",
- " dns_sev = int(rating_btn.selected_label) if not \"--Select--\" in query_select.value else \"\" \n",
- "\n",
- " if quick_text.value: \n",
- " ip = \"\"\n",
- " dns = quick_text.value\n",
- " dns_sev = int(rating_btn.selected_label) \n",
- " # Loop over every element in query_select widget\n",
- " score_values = []\n",
- " for query in query_select.options:\n",
- " if query.endswith(dns):\n",
- " # Matching element, create one row\n",
- " score_values.append((ip,query,ip_sev,dns_sev))\n",
- " else: \n",
- " ip = client_select.value if not \"--Select--\" in client_select.value else \"\"\n",
- " dns = query_select.value if not \"--Select--\" in query_select.value else \"\"\n",
- " score_values.append((ip,dns,ip_sev,dns_sev))\n",
- "\n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f, delimiter=',')\n",
- " rowct = 0\n",
- " with open(score_tmp, 'w') as score:\n",
- " wr = csv.DictWriter(score, delimiter=',', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames) \n",
- " wr.writeheader()\n",
- " for row in reader: \n",
- " for value in score_values: \n",
- " if row['ip_dst'] == value[0]: \n",
- " row['ip_sev'] = value[2] \n",
- " scored_threats.append(row) \n",
- " rowct += 1 \n",
- " break\n",
- " if row['dns_qry_name'] == value[1]: \n",
- " row['dns_sev'] = value[3] \n",
- " scored_threats.append(row) \n",
- " rowct += 1\n",
- " break\n",
- " wr.writerow(row) \n",
- " \n",
- " if not os.path.exists(score_fbk): \n",
- " with open(score_fbk, 'w') as feedback:\n",
- " wr = csv.DictWriter(feedback, delimiter='\\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames) \n",
- " wr.writeheader()\n",
- "\n",
- " with open(score_fbk, 'a') as feedback:\n",
- " for row in scored_threats:\n",
- " wr = csv.DictWriter(feedback, delimiter='\\t', quoting=csv.QUOTE_NONE, fieldnames=reader.fieldnames) \n",
- " wr.writerow(row)\n",
- "\n",
- " clear_output()\n",
- " print \"{0} matching connections scored\".format(rowct)\n",
- " !mv $score_tmp $sconnect \n",
- "\n",
- " if ip != \"--Select--\":\n",
- " display(Javascript(\"$(\\\"option[data-value='\" + ip +\"']\\\").remove();\"))\n",
- " if quick_text.value:\n",
- " display(Javascript(\"$(\\\"option[data-value$='\" + quick_text.value +\"']\\\").remove();\"))\n",
- " elif dns != \"--Select--\":\n",
- " display(Javascript(\"$(\\\"option[data-value='\" + dns +\"']\\\").remove();\"))\n",
- "\n",
- " client_select.value = \"--Select--\"\n",
- " query_select.value = \"--Select--\"\n",
- " quick_text.value = \"\"\n",
- "\n",
- "\n",
- "def save(b): \n",
- " clear_output() \n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- " data_loader() \n",
- " display(scoring_form)\n",
- " display(Javascript('reloadParentData();'))\n",
- " ml_feedback() \n",
- " print \"Suspicious connects successfully updated\"\n",
- "\n",
- "\n",
- "assign_btn.on_click(assign_score)\n",
- "save_btn.on_click(save)\n",
- " \n",
- "\n",
- "def ml_feedback():\n",
- " dst_name = os.path.basename(sconnect)\n",
- " str_fb=\"DSOURCE={0} &&\\\n",
- " FDATE={1} &&\\\n",
- " source /etc/spot.conf &&\\\n",
- " usr=$(echo $LUSER | cut -f3 -d'/') &&\\\n",
- " mlnode=$MLNODE &&\\\n",
- " lpath=$LPATH &&\\\n",
- " scp {2} $usr@$mlnode:$lpath/{3}\".format(dsource,date,score_fbk,dst_name) \n",
- "\n",
- " subprocess.call(str_fb, shell=True)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "# !cp $sconnectbu $sconnect"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.5"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/ipynb_templates/ThreatInvestigation.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/ipynb_templates/ThreatInvestigation.md b/spot-oa/oa/dns/ipynb_templates/ThreatInvestigation.md
index 281ff07..e58c871 100644
--- a/spot-oa/oa/dns/ipynb_templates/ThreatInvestigation.md
+++ b/spot-oa/oa/dns/ipynb_templates/ThreatInvestigation.md
@@ -1,6 +1,6 @@
-#DNS Threat Investigation Notebook
+# DNS Threat Investigation Notebook
-###Dependencies
+### Dependencies
- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html)
- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/)
- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip)
@@ -19,62 +19,33 @@ The following python modules will have to be imported for the notebook to work c
from IPython.html import widgets
from IPython.display import display, Javascript, clear_output
-##Pre-requisites
+## Pre-requisites
- Execution of the spot-oa process for DNS
-- Score a set connections in the Edge Investigation Notebook
-- Correct setup of the spot.conf file. [Read more](/wiki/Edit%20Solution%20Configuration)
+- Correct installation of the UI [Read more](/ui/INSTALL.md)
+- Score a set connections at the Edge Investigation Notebook
+- Correct setup the spot.conf file. [Read more](/wiki/Edit%20Solution%20Configuration)
-
-##Additional Configuration
+## Additional Configuration
`top_results` - This value defines the number of rows that will be displayed onscreen after the expanded search.
+## Data source
+Data should exists in the following tables:
+ *dns*
+ *dns_threat_investigation*
-##Data source
-The whole process in this notebook depends entirely on the existence of the scored `dns_scores.csv` file, which is generated at the OA process, and scored at the Edge Investigation Notebook.
-
-**Input files**
-All these paths should be relative to the main OA path.
-Schema for these files can be found [here](/spot-oa/oa/DNS)
-
- data/dns/<date>/dns_scores.csv
-
-**Output files**
-
-- threats.csv : Pipe separated file containing the comments saved by the user. This file is updated every time the user adds comments for a new threat.
-
- Schema with zero-indexed columns:
-
- 0.ip_dst : string
- 0.dns_qry_name : string
- 1.title: string
- 2.description: string
-
-- threat-dendro-\<anchor>.csv : Comma separated file generated in base of the results from the expanded
-search query. This file includes a list of connections involving the DNS or IP selected from the list.
-These results are limited to the day under analysis.
-
-
- Schema with zero-indexed columns:
+**Output**
+The following tables will be populated after the threat investigation process:
+ *dns_storyboard*
+ *dns_threat_dendro*
- 0.total: int
- 1.dns_qry_name: string
- 2.ip_dst: string
- 3.sev: int
+## FUNCTIONS
-
-**HDFS tables consumed**
-
- dns
-
-##FUNCTIONS
-
-**Widget configuration**
+### **Widget configuration**
This is not a function, but more like global code to set up styles and widgets to format the output of the notebook.
-`start_investigation():` - This function cleans the notebook from previous executions, then calls the data_loader() function to obtain the data and afterwards displays the corresponding widgets
+`start_investigation():` - This function cleans the notebook from previous executions.
-`data_loader():` - This function loads the _dns_scores.csv_ file into a csv dictionary reader to find all `ip_dst` values where `ip_sev` = 1, and the `dns_qry_name` where `dns_sev` = 1, merging both
-lists into a dictionary to populate the 'Suspicious DNS' listbox, through the _fill_list()_ function.
+`data_loader():` - , then calls the *threats* query to get the `ip_dst` and `dns_qry_name` values previously scored as high risk, merging both lists into a single dictionary to populate the 'Suspicious DNS' listbox, through the _fill_list()_ function.
`display_controls(ip_list):` - This function will only display the main widget box, containing:
- "Suspicious URI" listbox
@@ -82,19 +53,15 @@ lists into a dictionary to populate the 'Suspicious DNS' listbox, through the _f
- Container for the "Threat summary" and "Title" text boxes
- Container for the "Top N results" HTML table
-`fill_list(list_control,source):` - This function populates a listbox widget with the given data dictionary and appends an empty item at the top with the value '--Select--' (Just for visualization sake)
+`fill_list(list_control,source):` - This function populates a listbox widget with the given data dictionary and appends an empty item at the top with the value '--Select--' (Just for visualization sake)
-`search_ip(b):` - This function is triggered by the onclick event of the "Search" button. This will get the selected value from the listbox and perform a query to the _dns_ table to retrieve all comunication involving that IP/Domain during the day with any other IPs or Domains.
-The output of the query will automatically be stored in the _threat-dendro-<threat>.csv_ file.
-Afterwards it will read through the output file to display the HTML table, and the results displayed will be limited by the value set in the _top_results_ variable,
-ordered by amount of connections, listing the most active connections first.
+`search_ip(b):` - This function is triggered by the onclick event of the "Search" button. This calls the graphql *threat / details* query to find additional connections involving the selected IP or query name.
+The results will be displayed in the HTML table, ordered by amount of connections, listing the most active connections first.
Here the "display_threat_box()" function will be invoqued.
`display_threat_box(ip):` - Generates and displays the widgets for "Title" and "Comments" textboxes and the "Save" button on the notebook.
`save_threat_summary(b):` - This function is triggered by the _onclick_ event on the 'Save' button.
This will take the contents of the form and create/update the _threats.csv_ file.
-
-`file_is_empty(path):` - Performs a validation to check the file size to determine if it is empty.
-
+
`removeWidget(index):` - Javascript function that removes a specific widget from the notebook.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb b/spot-oa/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb
index 6eb9bad..fbfa976 100644
--- a/spot-oa/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb
+++ b/spot-oa/oa/dns/ipynb_templates/Threat_Investigation_master.ipynb
@@ -11,7 +11,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
@@ -30,28 +30,21 @@
" from IPython.html import widgets\n",
"from IPython.display import display, HTML, clear_output, Javascript \n",
"\n",
- "with open('/etc/spot.conf') as conf:\n",
- " for line in conf.readlines():\n",
- " if \"DBNAME=\" in line: DBNAME = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
- " elif \"IMPALA_DEM=\" in line: IMPALA_DEM = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
- "\n",
"path = os.getcwd().split(\"/\") \n",
"t_date = path[len(path)-1] \n",
- "dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
- "t_date = path[len(path)-1] \n",
- "sconnect = dpath + 'dns_scores.csv' \n",
- "threat_f = dpath + \"threats.csv\"\n",
+ "\n",
"anchor = ''\n",
"anchor_type = ''\n",
"top_results = 20\n",
- "details_limit = 1000"
+ "details_limit = 1000\n",
+ "query_comments = {}\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
@@ -127,7 +120,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": true
+ "collapsed": false
},
"outputs": [],
"source": [
@@ -144,29 +137,63 @@
" ips_query = {} \n",
" ip_sev={}\n",
" dns_sev={}\n",
- " c_ips=[]\n",
- " c_dns=[]\n",
- "\n",
- " if os.path.isfile(threat_f) and not file_is_empty(threat_f):\n",
- " with open(threat_f, 'r') as th:\n",
- " t_read = csv.reader(th, delimiter='|')\n",
- " t_read.next()\n",
- " for row in t_read: \n",
- " if row[0] != '' : c_ips.append(row[0])\n",
- " if row[1] != '' : c_dns.append(row[1])\n",
- " \n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f, delimiter=',')\n",
- " for row in reader:\n",
- " if row['ip_dst'] not in ips_query and row['ip_dst'] not in c_ips and row['ip_sev'] == '1': \n",
- " ips_query[row['ip_dst']]='i'\n",
- " if row['dns_qry_name'] not in ips_query and row['dns_qry_name'] not in c_dns and row['dns_sev'] == '1':\n",
- " ips_query[row['dns_qry_name']]='q' \n",
+ " global query_comments\n",
+ " \n",
+ " response = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " dns{\n",
+ " threats{\n",
+ " list(date:$date) {\n",
+ " dnsScore\n",
+ " clientIpScore\n",
+ " clientIp\n",
+ " dnsQuery\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(t_date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " )\n",
+ " \n",
+ " query_comments = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " dns{\n",
+ " threats{\n",
+ " comments(date:$date) { \n",
+ " title\n",
+ " text\n",
+ " ... on DnsQueryCommentType {\n",
+ " dnsQuery\n",
+ " }\n",
+ " ... on DnsClientIpCommentType {\n",
+ " clientIp\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(t_date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " )\n",
+ " query_comments = query_comments['data']['dns']['threats']['comments'] \n",
+ " \n",
+ " if not 'errors' in response: \n",
+ " for row in response['data']['dns']['threats']['list']: \n",
+ " if row['clientIp'] not in ips_query and row['clientIpScore'] == 1: \n",
+ " ips_query[row['clientIp']]='i'\n",
+ " if row['dnsQuery'] not in ips_query and row['dnsScore'] == 1: \n",
+ " ips_query[row['dnsQuery']]='q' \n",
" \n",
- " if row['ip_dst'] not in ip_sev: \n",
- " ip_sev[row['ip_dst']] = row['score']\n",
- " if row['dns_qry_name'] not in dns_sev: \n",
- " dns_sev[row['dns_qry_name']] =row['score']\n",
+ " if row['clientIp'] not in ip_sev: \n",
+ " ip_sev[row['clientIp']] = row['clientIpScore']\n",
+ " if row['dnsQuery'] not in dns_sev: \n",
+ " dns_sev[row['dnsQuery']] =row['dnsScore']\n",
+ " else:\n",
+ " print \"An error ocurred: \" + response[\"errors\"][0][\"message\"]\n",
+ " \n",
" \n",
" threat_title.value =\"<h4>Suspicious DNS</h4>\"\n",
" \n",
@@ -192,7 +219,8 @@
" \n",
" def search_ip(b): \n",
" global anchor \n",
- " global anchor_type\n",
+ " global anchor_type \n",
+ " global expanded_results\n",
" anchor = ''\n",
" anchor_type = ''\n",
" anchor = susp_select.selected_label \n",
@@ -200,57 +228,63 @@
" removeWidget(2)\n",
" removeWidget(1) \n",
" clear_output()\n",
- " \n",
- " global ir_f\n",
- " ir_f = dpath + 'threat-dendro-' + anchor + \".csv\" \n",
- " table = \"<table><th>IP</th><th>QUERY</th><th>TOTAL</th>\"\n",
- " \n",
- " if not os.path.isfile(ir_f) or (os.path.isfile(ir_f) and file_is_empty(ir_f)):\n",
- " if anchor_type == 'i':\n",
- " imp_query = \"\\\" SELECT COUNT(dns_qry_name) as total, dns_qry_name, ip_dst, 0 as sev FROM {0}.dns \\\n",
- " WHERE y={1} AND m={2} AND d={3} AND ip_dst='{4}' GROUP BY dns_qry_name, ip_dst \\\n",
- " ORDER BY total DESC LIMIT {5}\\\"\"\n",
- " elif anchor_type == 'q':\n",
- " imp_query = \"\\\" SELECT COUNT(ip_dst) as total, dns_qry_name, ip_dst, 0 as sev FROM {0}.dns \\\n",
- " WHERE y={1} AND m={2} AND d={3} AND dns_qry_name='{4}'\\\n",
- " GROUP BY ip_dst, dns_qry_name ORDER BY total DESC LIMIT {5}\\\"\"\n",
"\n",
- " imp_query=imp_query.format(DBNAME, yy, mm, dd, anchor, details_limit)\n",
- " !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
- " !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter=',' -q $imp_query -o $ir_f\n",
+ " \n",
+ " expanded_results = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType,$dnsQuery:String, $clientIp:SpotIpType){\n",
+ " dns{\n",
+ " threat{details(date:$date,dnsQuery:$dnsQuery,clientIp:$clientIp){\n",
+ " total \n",
+ " clientIp\n",
+ " dnsQuery \n",
+ " }}\n",
+ " }\n",
"\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(t_date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'dnsQuery': anchor if anchor_type == 'q' else None,\n",
+ " 'clientIp': anchor if anchor_type == 'i' else None \n",
+ " }\n",
+ " ) \n",
+ " \n",
" clear_output() \n",
- "# total, dns_qry_name, ip_dst, sev\n",
- " with open(ir_f, 'r') as f:\n",
- " try:\n",
- " reader = itertools.islice(csv.reader(f, delimiter=','), top_results) \n",
- " if reader!= '':\n",
- " reader.next()\n",
- " for row in reader: \n",
- " table += \"<tr><td class='spot-text-wrapper' data-toggle='tooltip'>\"+row[2]+\"</td>\\\n",
- " <td class='spot-text-wrapper' data-toggle='tooltip'>\"+row[1]+\"</td>\\\n",
- " <td align='center'>\"+str(row[0])+\"</td></tr>\" \n",
+ " \n",
+ " if not 'errors' in expanded_results:\n",
+ " table = \"<table><th>IP</th><th>QUERY</th><th>TOTAL</th>\"\n",
+ " \n",
+ " for row in expanded_results[\"data\"][\"dns\"][\"threat\"][\"details\"]: \n",
+ " table += \"<tr><td class='spot-text-wrapper' data-toggle='tooltip'>\"+row[\"clientIp\"]+\"</td>\\\n",
+ " <td class='spot-text-wrapper' data-toggle='tooltip'>\"+row[\"dnsQuery\"]+\"</td>\\\n",
+ " <td align='center'>\"+str(row[\"total\"])+\"</td></tr>\" \n",
"\n",
- " table += \"</table>\" \n",
- " result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results) \n",
- " except:\n",
- " table = \"<table></table>\"\n",
- " result_html_title.value='<h4>No results were found.</h4>'\n",
+ " table += \"</table>\" \n",
+ " result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results) \n",
+ " else:\n",
+ " print \"An error ocurred: \" + response[\"errors\"][0][\"message\"]\n",
"\n",
" result_html.value=table\n",
" result_html_box.children = [result_html]\n",
"\n",
- " display_threat_box(anchor)\n",
+ " display_threat_box(anchor,anchor_type)\n",
" resultTableBox.children = [result_html_title, result_html_box]\n",
" display(bottomBox)\n",
" \n",
" search_btn.on_click(search_ip)\n",
"\n",
" \n",
- "def display_threat_box(ip): \n",
+ "def display_threat_box(ip,anchor_type): \n",
+ " \n",
+ " global query_comments \n",
+ " title =\"\" \n",
+ " text = \"\"\n",
+ " data_filter = \"\"\n",
+ " data_filter = \"dnsQuery\" if anchor_type == 'q' else \"clientIp\"\n",
+ " title = next((item['title'] for item in query_comments if item.get(data_filter) == ip), \"\")\n",
+ " text = next((item['text'] for item in query_comments if item.get(data_filter) == ip), \"\") \n",
" result_title.value=\"<h4 class='spot-text-wrapper spot-text-xlg' data-toggle='tooltip'>Threat summary for \" + anchor +\"</h4>\"\n",
- " tc_txt_title = widgets.Text(value='', placeholder='Threat Title', width='100%')\n",
- " tc_txa_summary = widgets.Textarea(value='', height=100, width='95%')\n",
+ " tc_txt_title = widgets.Text(value=title, placeholder='Threat Title', width='100%')\n",
+ " tc_txa_summary = widgets.Textarea(value=text, height=100, width='95%')\n",
" tc_btn_save = widgets.Button(description='Save', width='65px', layout='width:100%')\n",
" tc_btn_save.button_style = 'primary'\n",
" \n",
@@ -263,26 +297,49 @@
" result_summary_box.children = [result_summary_container, result_button_container]\n",
" resultSummaryBox.children = [result_title,result_summary_box]\n",
" \n",
- " \n",
+ "\n",
" def save_threat_summary(b):\n",
" global anchor\n",
" anchor_ip =''\n",
" anchor_dns ='' \n",
- " if anchor != '': \n",
- " if anchor_type == 'i':\n",
- " anchor_ip = anchor\n",
- " elif anchor_type == 'q':\n",
- " anchor_dns = anchor\n",
- " \n",
- " if not os.path.exists(threat_f): \n",
- " with open(threat_f, 'w') as comment:\n",
- " comment.write('ip_dst|dns_qry_name|title|summary\\n')\n",
+ " \n",
+ " if anchor_type == 'i':\n",
+ " anchor_ip = anchor\n",
+ " elif anchor_type == 'q':\n",
+ " anchor_dns = anchor\n",
" \n",
- " with open(threat_f, 'a') as comment:\n",
- " comment.write(anchor_ip + '|' + anchor_dns + '|' + tc_txt_title.value + '|' +\n",
- " tc_txa_summary.value.replace('\\n', '\\\\n') + '\\n') \n",
+ " if anchor != '':\n",
+ " mutation=\"\"\"mutation(\n",
+ " $date: SpotDateType, \n",
+ " $dnsQuery:String, \n",
+ " $clientIp:SpotIpType,\n",
+ " $text: String!, \n",
+ " $title: String!,\n",
+ " $threatDetails: [DnsThreatDetailsInputType!]!) \n",
+ " {\n",
+ " dns{\n",
+ " createStoryboard(input:{\n",
+ " threatDetails: $threatDetails,\n",
+ " date: $date, \n",
+ " dnsQuery: $dnsQuery, \n",
+ " clientIp: $clientIp,\n",
+ " title: $title, \n",
+ " text: $text\n",
+ " })\n",
+ " {success}\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(t_date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'dnsQuery': anchor_dns if anchor_type == 'q' else None, \n",
+ " 'clientIp': anchor_ip if anchor_type == 'i' else None, \n",
+ " 'title': tc_txt_title.value,\n",
+ " 'text': tc_txa_summary.value.replace('\\n', '\\\\n'),\n",
+ " 'threatDetails': expanded_results['data']['dns']['threat']['details'] \n",
+ " }\n",
"\n",
- " display(Javascript(\"$(\\\"option[data-value='\" + anchor +\"']\\\").remove();\")) \n",
+ " response = GraphQLClient.request(mutation, variables) \n",
" display(Javascript(\"$('.widget-area > .widget-subarea > .widget-box:gt(0)').remove();\"))\n",
" \n",
" response = \"Summary successfully saved\"\n",
@@ -309,7 +366,7 @@
"cell_type": "code",
"execution_count": null,
"metadata": {
- "collapsed": false
+ "collapsed": false
},
"outputs": [],
"source": [
@@ -338,4 +395,4 @@
},
"nbformat": 4,
"nbformat_minor": 0
-}
\ No newline at end of file
+}
[09/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/README.md b/spot-oa/oa/flow/README.md
index 4b3dbc1..f027376 100644
--- a/spot-oa/oa/flow/README.md
+++ b/spot-oa/oa/flow/README.md
@@ -1,6 +1,6 @@
# **Flow OA**
-Flow sub-module extracts and transforms Flow data already ranked by spot-ml and will load into csv files for presentation layer.
+Flow sub-module extracts and transforms Flow data already ranked by spot-ml and will load into impala tables for presentation the layer.
## **Flow OA Components**
@@ -13,12 +13,16 @@ Flow spot-oa main script executes the following steps:
ipython Notebooks: ipynb/flow/<date>/
2. Creates a copy of iPython notebooks out of templates in ipynb_templates folder into output folder.
+
3. Reads Flow spot-ml results for a given date and loads only the requested limit.
+
4. Add network context to source and destination IPs.
+
5. Add geolocation to source and destination IPs.
- 6. Saves transformed data into a new csv file, this file is called flow_scores.csv.
- 7. Creates details, and chord diagram files. These details include information about each suspicious connection and some additional information
- to draw chord diagrams.
+
+ 6. Stores transformed data in the selected database.
+
+ 7. Generates details and chord diagram data. These details include information about aditional connections and some additional information to draw chord diagrams in the UI.
**Dependencies**
@@ -48,37 +52,37 @@ Before running Flow OA users need to configure components for the first time. It
**Output**
-- flow_scores.csv. Main results file for Flow OA. This file will contain suspicious connects information and it's limited to the number of rows the user selected when running [oa/start_oa.py](/spot-oa/oa/INSTALL.md#usage).
+- Flow suspicious connections. _flow\_scores_ table.
+
+Main results for Flow OA. The data stored in this table is limited by the number of rows the user selected when running [oa/start_oa.py](/spot-oa/oa/INSTALL.md#usage).
- Schema with zero-indexed columns:
- 0. sev: int
- 1. tstart: string
- 2. srcIP: string
- 3. dstIP: string
- 4. sport: int
- 5. dport: int
- 6. proto: string
- 7. ipkt: bigint
- 8. ibyt: bigint
- 9. opkt: bigint
- 10. obyt: bigint
- 11. score: double
- 12. rank: int
- 13. srcIpInternal: bit
- 14. destIpInternal: bit
- 15. srcGeo: string
- 16. dstGeo: string
- 17. srcDomain: string
- 18. dstDomain: string
- 19. srcIP_rep: string
- 20. dstIP_rep: string
-
-
-- flow_scores_bu.csv. Backup file for flow_scores.csv in case user needs to roll back the scoring or the changes made during analysis. Schema it's same as flow_scores.csv.
-
-- edge-\<source IP>-\<destination IP>-\<HH>-\<MM>.tsv. Edge files. One for each suspicious connection containing the details for each comunication occurred during the same specific minute between source IP and destination IP.
-
- Schema with zero-indexed columns:
+ Table schema:
+ 0. tstart: string
+ 1. srcip: string
+ 2. dstip: string
+ 3. sport: int
+ 4. dport: int
+ 5. proto: string
+ 6. ipkt: int
+ 7. ibyt: int
+ 8. opkt: int
+ 9. obyt: int
+ 10. score: float
+ 11. rank: int
+ 12. srcip_internal: bit
+ 13. destip_internal: bit
+ 15. src_geoloc: string
+ 16. dst_geoloc: string
+ 17. src_domain: string
+ 18. dst_domain: string
+ 19. src_rep: string
+ 20. dst_rep: string
+
+- Flow details. _flow\_edge_ table.
+
+A query will be executed for each suspicious connection detected, to find the details for each connection occurred during the same specific minute between given source IP and destination IP.
+
+ Table schema:
0. tstart: string
1. srcip: string
2. dstip: string
@@ -87,37 +91,57 @@ Before running Flow OA users need to configure components for the first time. It
5. proto: string
6. flags: string
7. tos: int
- 8. bytes: bigint
- 9. pkts: bigint
- 10. input: int
- 11. output: int
- 12. rip: string
-
-- chord-\<client ip>.tsv. Chord files. One for each distinct client ip. These files contain the sum of input packets and bytes transferred between the client ip and every other IP it connected to.
-
- Schema with zero-indexed columns:
- 0. srcip: string
- 1. dstip: string
- 2. ibytes: bigint
- 3. ipkts: double
-
+ 8. ibyt: bigint
+ 9. ipkt: bigint
+ 10. pkts: bigint
+ 11. input: int
+ 12. output: int
+ 13. rip: string
+ 14. obyt: bigint
+ 15. opkt: bigint
+ 16. hh: int
+ 17. md: int
+
+- Flow Chord Diagrams. _flow\_chords_ table.
+
+A query will be executed for each distinct client ip that has connections to 2 or more other suspicious IP. This query will retrieve the sum of input packets and bytes transferred between the client ip and every other suspicious IP it connected to.
+
+ Table schema:
+ 0. ip_threat: string
+ 1. srcip: string
+ 2. dstip: string
+ 3. ibyt: bigint
+ 4. ipkt: bigint
+
+
+- Flow Ingest summary. _flow\_ingest\_summary_ table.
+
+This table is populated with the number of connections ingested by minute during that day.
+
+ Table schema:
+ 0. tdate: string
+ 1. total: bigint
+
+
### flow_config.json
Flow spot-oa configuration. Contains columns name and index for input and output files.
This Json file contains 3 main arrays:
- flow_results_fields: list of column name and index of ML flow_results.csv file. Flow OA uses this mapping to reference columns by name.
- - column_indexes_filter: the list of indices to take out of flow_results_fields for OA process.
+ - column_indexes_filter: the list of indices to take out of flow_results_fields for the OA process.
- flow_score_fields: list of column name and index for flow_scores.csv. After the OA process completes more columns are added.
-
-
+
### ipynb_templates
Templates for iPython notebooks.
After OA process completes, a copy of each iPython notebook is going to be copied to the ipynb/\<pipeline>/\<date> path.
With these iPython notebooks user will be able to perform further analysis and score connections. User can also
experiment adding or modifying the code.
-If a new functionality is required for the ipython notebook, the templates need to be modified to include the functionality for new executions.
-For further reference on how to work with these notebooks, you can read:
-- [Edge Notebook](/spot-oa/oa/flow/ipynb_templates/EdgeNotebook.md)
-- [Threat Investigation Notebook](/spot-oa/oa/flow/ipynb_templates/ThreatInvestigation.md)
\ No newline at end of file
+If a new functionality is required for the ipython notebook, the templates need to be modified to include the functionality for posterior executions.
+For further reference on how to work with this notebook, you can read:
+- [Threat Investigation Notebook](/spot-oa/oa/flow/ipynb_templates/ThreatInvestigation.md)
+
+
+### Reset scored connections
+To reset all scored connections for a day, a specific cell with a preloaded function is included in the Advanced Mode Notebook. The cell is commented to avoid accidental executions, but is properly labeled.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/__init__.py b/spot-oa/oa/flow/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/flow/__init__.py
+++ b/spot-oa/oa/flow/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/flow_conf.json
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/flow_conf.json b/spot-oa/oa/flow/flow_conf.json
index 18e76a7..cd96b08 100644
--- a/spot-oa/oa/flow/flow_conf.json
+++ b/spot-oa/oa/flow/flow_conf.json
@@ -21,20 +21,19 @@
,"score":17
},
"column_indexes_filter": [0,8,9,10,11,12,13,14,15,16,17],
- "flow_score_fields": {
- "sev": 0
- ,"tstart": 1
- ,"srcIP": 2
- ,"dstIP":3
- ,"sport":4
- ,"dport":5
- ,"proto":6
- ,"ipkt":7
- ,"ibyt":8
- ,"opkt":9
- ,"obyt":10
- ,"score":11
- ,"rank":12
+ "flow_score_fields": {
+ "tstart": 0
+ ,"srcIP": 1
+ ,"dstIP":2
+ ,"sport":3
+ ,"dport":4
+ ,"proto":5
+ ,"ipkt":6
+ ,"ibyt":7
+ ,"opkt":8
+ ,"obyt":9
+ ,"score":10
+ ,"rank":11
},
"flow_feedback_fields": {
"sev":0
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/flow_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/flow_oa.py b/spot-oa/oa/flow/flow_oa.py
index 027b54f..53cec6b 100644
--- a/spot-oa/oa/flow/flow_oa.py
+++ b/spot-oa/oa/flow/flow_oa.py
@@ -22,27 +22,30 @@ import sys
import json
import numpy as np
import linecache, bisect
-import csv
+import csv, math
import pandas as pd
+import subprocess
+import numbers
+import api.resources.hdfs_client as HDFSClient
+import api.resources.impala_engine as impala
from collections import OrderedDict
from multiprocessing import Process
-from utils import Util,ProgressBar
+from utils import Util, ProgressBar
from components.data.data import Data
from components.geoloc.geoloc import GeoLocalization
from components.reputation.gti import gti
-
+from impala.util import as_pandas
import time
class OA(object):
- def __init__(self,date,limit=500,logger=None):
-
- self._initialize_members(date,limit,logger)
-
- def _initialize_members(self,date,limit,logger):
-
+ def __init__(self,date,limit=500,logger=None):
+ self._initialize_members(date,limit,logger)
+
+ def _initialize_members(self,date,limit,logger):
+
# get logger if exists. if not, create new instance.
self._logger = logging.getLogger('OA.Flow') if logger else Util.get_logger('OA.Flow',create_file=False)
@@ -57,31 +60,32 @@ class OA(object):
self._ingest_summary_path = None
self._flow_scores = []
self._results_delimiter = '\t'
+
# get app configuration.
self._spot_conf = Util.get_spot_conf()
- # get scores fields conf
+ # # get scores fields conf
conf_file = "{0}/flow_conf.json".format(self._scrtip_path)
- self._conf = json.loads(open (conf_file).read(),object_pairs_hook=OrderedDict)
-
+ self._conf = json.loads(open (conf_file).read(),object_pairs_hook=OrderedDict)
+
# initialize data engine
- self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '')
- self._engine = Data(self._db, self._table_name,self._logger)
-
+ self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '')
+
def start(self):
####################
start = time.time()
- ####################
+ ####################
self._create_folder_structure()
+ self._clear_previous_executions()
self._add_ipynb()
self._get_flow_results()
self._add_network_context()
self._add_geo_localization()
self._add_reputation()
- self._create_flow_scores_csv()
+ self._create_flow_scores()
self._get_oa_details()
self._ingest_summary()
@@ -89,19 +93,38 @@ class OA(object):
end = time.time()
print(end - start)
##################
-
- def _create_folder_structure(self):
+
+
+ def _clear_previous_executions(self):
+
+ self._logger.info("Cleaning data from previous executions for the day")
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ table_schema = []
+ HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
+ table_schema=['suspicious', 'edge','chords','threat_investigation', 'timeline', 'storyboard', 'summary' ]
+
+ for path in table_schema:
+ HDFSClient.delete_folder("{0}/{1}/hive/oa/{2}/y={3}/m={4}/d={5}".format(HUSER,self._table_name,path,yr,int(mn),int(dy)),user="impala")
+ impala.execute_query("invalidate metadata")
+ #removes Feedback file
+ HDFSClient.delete_folder("{0}/{1}/scored_results/{2}{3}{4}/feedback/ml_feedback.csv".format(HUSER,self._table_name,yr,mn,dy))
+ #removes json files from the storyboard
+ HDFSClient.delete_folder("{0}/{1}/oa/{2}/{3}/{4}/{5}".format(HUSER,self._table_name,"storyboard",yr,mn,dy))
+
+ def _create_folder_structure(self):
- # create date folder structure if it does not exist.
self._logger.info("Creating folder structure for OA (data and ipynb)")
self._data_path,self._ingest_summary_path,self._ipynb_path = Util.create_oa_folders("flow",self._date)
+
def _add_ipynb(self):
if os.path.isdir(self._ipynb_path):
- self._logger.info("Adding edge investigation IPython Notebook")
- shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+ self._logger.info("Adding the advanced mode IPython Notebook")
+ shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
self._logger.info("Adding threat investigation IPython Notebook")
shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
@@ -109,6 +132,7 @@ class OA(object):
else:
self._logger.error("There was a problem adding the IPython Notebooks, please check the directory exists.")
+
def _get_flow_results(self):
self._logger.info("Getting {0} Machine Learning Results from HDFS".format(self._date))
@@ -117,8 +141,8 @@ class OA(object):
# get hdfs path from conf file
HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
hdfs_path = "{0}/flow/scored_results/{1}/scores/flow_results.csv".format(HUSER,self._date)
-
- # get results file from hdfs
+
+ # get results file from hdfs
get_command = Util.get_ml_results_form_hdfs(hdfs_path,self._data_path)
self._logger.info("{0}".format(get_command))
@@ -134,36 +158,36 @@ class OA(object):
self._logger.error("There was an error getting ML results from HDFS")
sys.exit(1)
- # add headers.
- self._logger.info("Adding headers based on configuration file: score_fields.json")
- self._flow_scores = [ [ str(key) for (key,value) in self._conf['flow_score_fields'].items()] ]
-
- # filter results add sev and rank.
+ # filter results add rank.
self._logger.info("Filtering required columns based on configuration")
- self._flow_scores.extend([ [0] + [ conn[i] for i in self._conf['column_indexes_filter'] ] + [n] for n, conn in enumerate(self._flow_results) ])
+
+ self._flow_scores.extend([ [ conn[i] for i in self._conf['column_indexes_filter'] ] + [n] for n, conn in enumerate(self._flow_results) ])
- def _create_flow_scores_csv(self):
- flow_scores_csv = "{0}/flow_scores.csv".format(self._data_path)
- Util.create_csv_file(flow_scores_csv,self._flow_scores)
+ def _create_flow_scores(self):
- # create bk file
- flow_scores_bu_csv = "{0}/flow_scores_bu.csv".format(self._data_path)
- Util.create_csv_file(flow_scores_bu_csv,self._flow_scores)
+ # get date parameters.
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ value_string = ""
+
+ for row in self._flow_scores:
+ value_string += str(tuple(Util.cast_val(item) for item in row)) + ","
+
+ load_into_impala = ("""
+ INSERT INTO {0}.flow_scores partition(y={2}, m={3}, d={4}) VALUES {1}
+ """).format(self._db, value_string[:-1], yr, mn, dy)
+ impala.execute_query(load_into_impala)
+
def _add_network_context(self):
# use ipranges to see if the IPs are internals.
ip_ranges_file = "{0}/context/ipranges.csv".format(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
- # add new headers (srcIpInternal/destIpInternal).
- self._logger.info("Adding network context headers")
- flow_headers = self._flow_scores[0]
- flow_headers.extend(["srcIpInternal","destIpInternal"])
-
# add values to srcIpInternal and destIpInternal.
flow_scores = iter(self._flow_scores)
- next(flow_scores)
if os.path.isfile(ip_ranges_file):
@@ -184,11 +208,9 @@ class OA(object):
self._flow_scores = [ conn + [ self._is_ip_internal(conn[src_ip_index],ip_internal_ranges)]+[ self._is_ip_internal(conn[dst_ip_index],ip_internal_ranges)] for conn in flow_scores]
else:
-
- self._flow_scores = [ conn + ["",""] for conn in flow_scores ]
+ self._flow_scores = [ conn + [0,0] for conn in flow_scores ]
self._logger.info("WARNING: Network context was not added because the file ipranges.csv does not exist.")
- self._flow_scores.insert(0,flow_headers)
def _is_ip_internal(self,ip, ranges):
result = 0
@@ -204,14 +226,10 @@ class OA(object):
# use ipranges to see if the IPs are internals.
iploc_file = "{0}/context/iploc.csv".format(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
- # add new headers (srcIpInternal/destIpInternal).
self._logger.info("Adding geo localization headers")
- flow_headers = self._flow_scores[0]
- flow_headers.extend(["srcGeo","dstGeo","srcDomain","dstDomain"])
# add values to srcIpInternal and destIpInternal.
flow_scores = iter(self._flow_scores)
- next(flow_scores)
if os.path.isfile(iploc_file):
@@ -241,17 +259,11 @@ class OA(object):
self._flow_scores = [ conn + ["","","",""] for conn in flow_scores ]
self._logger.info("WARNING: IP location was not added because the file {0} does not exist.".format(iploc_file))
- self._flow_scores.insert(0,flow_headers)
-
+
def _add_reputation(self):
reputation_conf_file = "{0}/components/reputation/reputation_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
- # add new headers (gtiSrcRep/gtiDstRep).
- self._logger.info("Adding reputation headers")
- flow_headers_rep = self._flow_scores[0]
- flow_headers_rep.extend(["srcIP_rep","dstIP_rep"])
-
# read configuration.
self._logger.info("Reading reputation configuration file: {0}".format(reputation_conf_file))
rep_conf = json.loads(open(reputation_conf_file).read())
@@ -268,7 +280,6 @@ class OA(object):
self._logger.info("Getting GTI reputation for src IPs")
flow_scores_src = iter(self._flow_scores)
- next(flow_scores_src)
# getting reputation for src IPs
src_ips = [ conn[src_ip_index] for conn in flow_scores_src ]
@@ -276,30 +287,25 @@ class OA(object):
self._logger.info("Getting GTI reputation for dst IPs")
flow_scores_dst = iter(self._flow_scores)
- next(flow_scores_dst)
# getting reputation for dst IPs
dst_ips = [ conn[dst_ip_index] for conn in flow_scores_dst ]
dst_rep_results = flow_gti.check(dst_ips)
flow_scores_final = iter(self._flow_scores)
- next(flow_scores_final)
self._flow_scores = []
- flow_scores = [conn + [src_rep_results[conn[src_ip_index]]] + [dst_rep_results[conn[dst_ip_index]]] for conn in flow_scores_final ]
+ flow_scores = [conn + [src_rep_results[conn[src_ip_index]]] + [dst_rep_results[conn[dst_ip_index]]] for conn in flow_scores_final ]
self._flow_scores = flow_scores
else:
# add values to gtiSrcRep and gtiDstRep.
flow_scores = iter(self._flow_scores)
- next(flow_scores)
self._flow_scores = [ conn + ["",""] for conn in flow_scores ]
self._logger.info("WARNING: IP reputation was not added. No refclient configured")
- self._flow_scores.insert(0,flow_headers_rep)
-
def _get_oa_details(self):
self._logger.info("Getting OA Flow suspicious details/chord diagram")
@@ -319,8 +325,6 @@ class OA(object):
# skip header
sp_connections = iter(self._flow_scores)
- next(sp_connections)
-
# loop connections.
connections_added = []
for conn in sp_connections:
@@ -330,7 +334,7 @@ class OA(object):
continue
else:
connections_added.append(conn)
-
+
src_ip_index = self._conf["flow_score_fields"]["srcIP"]
dst_ip_index = self._conf["flow_score_fields"]["dstIP"]
@@ -340,34 +344,32 @@ class OA(object):
dip = conn[dst_ip_index]
# get hour and date (i.e. 2014-07-08 10:10:40)
- date_array = conn[1].split(' ')
+
+ date_array = conn[0].split(' ')
date_array_1 = date_array[0].split('-')
date_array_2 = date_array[1].split(':')
-
+
yr = date_array_1[0]
dy = date_array_1[2]
mh = date_array_1[1]
hr = date_array_2[0]
mm = date_array_2[1]
-
- # connection details query.
- sp_query = ("SELECT treceived as tstart,sip as srcip,dip as dstip,sport as sport,dport as dport,proto as proto,flag as flags,stos as TOS,ibyt as ibytes,ipkt as ipkts,input as input, output as output,rip as rip, obyt as obytes, opkt as opkts from {0}.{1} where ((sip='{2}' AND dip='{3}') or (sip='{3}' AND dip='{2}')) AND y={8} AND m={4} AND d={5} AND h={6} AND trminute={7} order by tstart limit 100")
-
- # sp query.
- sp_query = sp_query.format(self._db,self._table_name,sip,dip,mh,dy,hr,mm,yr)
-
- # output file.
- edge_file = "{0}/edge-{1}-{2}-{3}-{4}.tsv".format(self._data_path,sip.replace(".","_"),dip.replace(".","_"),hr,mm)
+
+ query_to_load = ("""
+ INSERT INTO TABLE {0}.flow_edge PARTITION (y={2}, m={3}, d={4})
+ SELECT treceived as tstart,sip as srcip,dip as dstip,sport as sport,dport as dport,proto as proto,flag as flags,
+ stos as tos,ibyt as ibyt,ipkt as ipkt, input as input, output as output,rip as rip, obyt as obyt,
+ opkt as opkt, h as hh, trminute as mn from {0}.{1} where ((sip='{7}' AND dip='{8}') or (sip='{8}' AND dip='{7}'))
+ AND y={2} AND m={3} AND d={4} AND h={5} AND trminute={6};
+ """).format(self._db,self._table_name,yr, mh, dy, hr, mm, sip,dip)
+ impala.execute_query(query_to_load)
+
- # execute query
- self._engine.query(sp_query,output_file=edge_file,delimiter="\\t")
-
def _get_chord_details(self,bar=None):
# skip header
sp_connections = iter(self._flow_scores)
- next(sp_connections)
src_ip_index = self._conf["flow_score_fields"]["srcIP"]
dst_ip_index = self._conf["flow_score_fields"]["dstIP"]
@@ -389,21 +391,24 @@ class OA(object):
if n > 1:
ip_list = []
sp_connections = iter(self._flow_scores)
- next(sp_connections)
for row in sp_connections:
- if ip == row[2] : ip_list.append(row[3])
- if ip == row[3] :ip_list.append(row[2])
+ if ip == row[1] : ip_list.append(row[2])
+ if ip == row[2] :ip_list.append(row[1])
ips = list(set(ip_list))
if len(ips) > 1:
ips_filter = (",".join(str("'{0}'".format(ip)) for ip in ips))
- chord_file = "{0}/chord-{1}.tsv".format(self._data_path,ip.replace(".","_"))
- ch_query = ("SELECT sip as srcip, dip as dstip, SUM(ibyt) as ibytes, SUM(ipkt) as ipkts from {0}.{1} where y={2} and m={3} \
- and d={4} and ( (sip='{5}' and dip IN({6})) or (sip IN({6}) and dip='{5}') ) group by sip,dip")
- self._engine.query(ch_query.format(self._db,self._table_name,yr,mn,dy,ip,ips_filter),chord_file,delimiter="\\t")
+
+ query_to_load = ("""
+ INSERT INTO TABLE {0}.flow_chords PARTITION (y={2}, m={3}, d={4})
+ SELECT '{5}' as ip_threat, sip as srcip, dip as dstip, SUM(ibyt) as ibyt, SUM(ipkt) as ipkt from {0}.{1} where y={2} and m={3}
+ and d={4} and ((sip='{5}' and dip IN({6})) or (sip IN({6}) and dip='{5}')) group by sip,dip,m,d;
+ """).format(self._db,self._table_name,yr,mn,dy,ip,ips_filter)
-
- def _ingest_summary(self):
+ impala.execute_query(query_to_load)
+
+
+ def _ingest_summary(self):
# get date parameters.
yr = self._date[:4]
mn = self._date[4:6]
@@ -412,46 +417,52 @@ class OA(object):
self._logger.info("Getting ingest summary data for the day")
ingest_summary_cols = ["date","total"]
- result_rows = []
- df_filtered = pd.DataFrame()
-
- ingest_summary_file = "{0}/is_{1}{2}.csv".format(self._ingest_summary_path,yr,mn)
- ingest_summary_tmp = "{0}.tmp".format(ingest_summary_file)
- if os.path.isfile(ingest_summary_file):
- df = pd.read_csv(ingest_summary_file, delimiter=',',names=ingest_summary_cols, skiprows=1)
- df_filtered = df[df['date'].str.contains("{0}-{1}-{2}".format(yr, mn, dy)) == False]
- else:
- df = pd.DataFrame()
+ result_rows = []
+ df_filtered = pd.DataFrame()
+
+ # get ingest summary.
+
+ query_to_load=("""
+ SELECT tryear, trmonth, trday, trhour, trminute, COUNT(*) as total
+ FROM {0}.{1} WHERE y={2} AND m={3} AND d={4}
+ AND unix_tstamp IS NOT NULL
+ AND sip IS NOT NULL
+ AND sport IS NOT NULL
+ AND dip IS NOT NULL
+ AND dport IS NOT NULL
+ AND ibyt IS NOT NULL
+ AND ipkt IS NOT NULL
+ AND tryear={2}
+ AND cast(treceived as timestamp) IS NOT NULL
+ GROUP BY tryear, trmonth, trday, trhour, trminute;
+ """).format(self._db,self._table_name, yr, mn, dy)
- # get ingest summary.
- ingest_summary_qry = ("SELECT tryear, trmonth, trday, trhour, trminute, COUNT(*) total"
- " FROM {0}.{1} "
- " WHERE "
- " y={2} "
- " AND m={3} "
- " AND d={4} "
- " AND unix_tstamp IS NOT NULL AND sip IS NOT NULL "
- " AND sport IS NOT NULL AND dip IS NOT NULL "
- " AND dport IS NOT NULL AND ibyt IS NOT NULL "
- " AND ipkt IS NOT NULL "
- " GROUP BY tryear, trmonth, trday, trhour, trminute;")
-
-
- ingest_summary_qry = ingest_summary_qry.format(self._db,self._table_name, yr, mn, dy)
- results_file = "{0}/results_{1}.csv".format(self._ingest_summary_path,self._date)
- self._engine.query(ingest_summary_qry,output_file=results_file,delimiter=",")
-
- if os.path.isfile(results_file):
- result_rows = pd.read_csv(results_file, delimiter=',')
-
- df_new = pd.DataFrame([["{0}-{1}-{2} {3}:{4}".format(yr, mn, dy, str(val['trhour']).zfill(2), str(val['trminute']).zfill(2)), int(val[5])] for key,val in result_rows.iterrows()],columns = ingest_summary_cols)
-
- df_filtered = df_filtered.append(df_new, ignore_index=True)
- df_filtered.to_csv(ingest_summary_tmp,sep=',', index=False)
-
- os.remove(results_file)
- os.rename(ingest_summary_tmp,ingest_summary_file)
+ results = impala.execute_query(query_to_load)
+
+ if results:
+ df_results = as_pandas(results)
+
+ #Forms a new dataframe splitting the minutes from the time column
+ df_new = pd.DataFrame([["{0}-{1}-{2} {3}:{4}".format(val['tryear'],val['trmonth'],val['trday'], val['trhour'], val['trminute']), int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df_results.iterrows()],columns = ingest_summary_cols)
+ value_string = ''
+ #Groups the data by minute
+
+ sf = df_new.groupby(by=['date'])['total'].sum()
+ df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})
+
+ df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False)
+ if len(df_final) > 0:
+ query_to_insert=("""
+ INSERT INTO {0}.flow_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
+ """).format(self._db, yr, mn, dy, tuple(df_final))
+
+ impala.execute_query(query_to_insert)
+
else:
self._logger.info("No data found for the ingest summary")
-
\ No newline at end of file
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb b/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb
new file mode 100644
index 0000000..55f5e70
--- /dev/null
+++ b/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb
@@ -0,0 +1,292 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Apache Spot's Ipython Advanced Mode\n",
+ "## Netflows\n",
+ "This guide provides examples about how to request data, show data with some cool libraries like pandas and more.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Import Libraries**\n",
+ "\n",
+ "The next cell will import the necessary libraries to execute the functions. Do not remove"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import linecache, bisect\n",
+ "import os\n",
+ "\n",
+ "spath = os.getcwd()\n",
+ "path = spath.split(\"/\")\n",
+ "date = path[len(path)-1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Request Data**\n",
+ "\n",
+ "In order to request data we are using Graphql (a query language for APIs, more info at: http://graphql.org/).\n",
+ "\n",
+ "We provide the function to make a data request, all you need is a query and variables\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "def makeGraphqlRequest(query, variables):\n",
+ " return GraphQLClient.request(query, variables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now that we have a function, we can run a query like this:\n",
+ "\n",
+ "*Note: There's no need to manually set the date for the query, by default the code will read the date from the current path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "suspicious_query = \"\"\"query($date:SpotDateType) {\n",
+ " flow {\n",
+ " suspicious(date:$date)\n",
+ " {\n",
+ " srcIp\n",
+ " dstIp\n",
+ " srcPort\n",
+ " dstPort\n",
+ " score\n",
+ " srcIp_domain\n",
+ " dstIp_rep\n",
+ " protocol\n",
+ " outBytes\n",
+ " inPkts\n",
+ " srcIp_rep\n",
+ " inBytes\n",
+ " srcIp_isInternal \n",
+ " rank \n",
+ " dstIp_geoloc\n",
+ " tstart\n",
+ " outPkts \n",
+ " dstIp_isInternal\n",
+ " dstIp_domain\n",
+ " }\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ "##If you want to use a different date for your query, switch the \n",
+ "##commented/uncommented following lines\n",
+ "\n",
+ "variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# 'date': \"2016-10-08\"\n",
+ " }\n",
+ " \n",
+ "suspicious_request = makeGraphqlRequest(suspicious_query,variables)\n",
+ "\n",
+ "##The variable suspicious_request will contain the resulting data from the query.\n",
+ "results = suspicious_request['data']['flow']['suspicious']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Pandas Dataframes\n",
+ "\n",
+ "The following cell loads the results into a pandas dataframe\n",
+ "\n",
+ "For more information on how to use pandas, you can learn more here: https://pandas.pydata.org/pandas-docs/stable/10min.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "df = pd.read_json(json.dumps(results))\n",
+ "##Printing only the selected column list from the dataframe\n",
+ "##By default it will only print the first 15 results\n",
+ "print df[['srcIp','dstIp','srcPort','dstPort','score']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Additional operations \n",
+ "\n",
+ "Additional operations can be performed on the dataframe like sorting the data, filtering it and grouping it\n",
+ "\n",
+ "**Filtering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "##Filter results where the destination port = 3389\n",
+ "##The resulting data will be stored in df2 \n",
+ "\n",
+ "df2 = df[df['dstPort'].isin(['3389'])]\n",
+ "print df2[['srcIp','dstIp','srcPort','dstPort','score']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Ordering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "srtd = df.sort_values(by=\"rank\")\n",
+ "print srtd[['rank','srcIp','dstIp','srcPort','dstPort','score']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Grouping the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "## This command will group the results by pairs of source-destination IP\n",
+ "## summarizing all other columns \n",
+ "grpd = df.groupby(['srcIp','dstIp']).sum()\n",
+ "## This will print the resulting dataframe displaying the input and output bytes columnns\n",
+ "print grpd[[\"inBytes\",\"inPkts\"]]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Reset Scored Connections**\n",
+ "\n",
+ "Uncomment and execute the following cell to reset all scored connections for this day"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "# reset_scores = \"\"\"mutation($date:SpotDateType!) {\n",
+ "# flow{\n",
+ "# resetScoredConnections(date:$date){\n",
+ "# success\n",
+ "# }\n",
+ "# }\n",
+ "# }\"\"\"\n",
+ "\n",
+ "\n",
+ "# variables={\n",
+ "# 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# }\n",
+ " \n",
+ "# request = makeGraphqlRequest(reset_scores,variables)\n",
+ "\n",
+ "# print request['data']['flow']['resetScoredConnections ']['success']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Sandbox\n",
+ "\n",
+ "At this point you can perform your own analysis using the previously provided functions as a guide.\n",
+ "\n",
+ "Happy threat hunting!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "#Your code here"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.10"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/ipynb_templates/EdgeNotebook.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/ipynb_templates/EdgeNotebook.md b/spot-oa/oa/flow/ipynb_templates/EdgeNotebook.md
deleted file mode 100644
index 272e2bc..0000000
--- a/spot-oa/oa/flow/ipynb_templates/EdgeNotebook.md
+++ /dev/null
@@ -1,97 +0,0 @@
-#Flow Edge Investigation Notebook
-
-###Dependencies
-- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html)
-- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/)
-- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip)
-- [pandas](http://pandas.pydata.org/)
-
-The following python modules will be imported for the notebook to work correctly:
-
- import struct, socket
- import shutil
- import numpy as np
- import pandas as pd
- import linecache, bisect
- import csv
- import operator
- import os, time, subprocess
- import ipywidgets #For jupyter/ipython >= 1.4
- from IPython.html import widgets #For jupyter/ipython < 1.4
- from IPython.display import display, HTML, clear_output, Javascript
-
-
-###Pre-requisites
-- Execution of the spot-oa process for Flow
-- Correct setup the spot.conf file. [Read more](/wiki/Edit%20Solution%20Configuration)
-- Have a public key created between the current UI node and the ML node. [Read more](/wiki/Configure%20User%20Accounts#configure-user-accounts)
-
-
-##Additional Configuration
-`coff` - This value defines the max number of records used to populate the listbox widgets. This value is set by default on 250.
-`nwloc` - File name of the custom network context.
-
-###Data source
-The whole process in this notebook depends entirely on the existence of `flow_scores.csv` file, which is generated at the OA process at the path.
-The data is directly manipulated on the .csv files, so a `flow_scores_bu.csv` on the same path is created as a backup to allow the user to restore the original data at any point,
-and this can be performed executing the last cell on the notebook with the following command:
-
- !cp $sconnectbu $sconnect
-
-
-**Input files**
-All these paths should be relative to the main OA path.
-Schema for these files can be found [here](/spot-oa/oa/flow)
-
- data/flow/<date>/flow_scores.csv
- data/flow/<date>/flow_scores_bu.csv
-
-**Temporary Files**
-
- data/flow/<date>/flow_scores.csv.tmp
-
-**Output files**
-
- data/flow/<date>/flow_scores.csv (Updated with severity values)
- data/flow/<date>/flow_scores_fb.csv (File with scored connections that will be used for ML feedback)
-
-##Functions
-
-`displaythis():` - This function reads the `flow_scores.csv` file to list all suspicious unscored connections, creating separated lists for:
-- Source IP
-- Destination IP
-- Source port
-- Destination port
-
-Each of these lists will populate a listbox widget and then they will be displayed to help the user narrow down the selection and score more specific connections by combining the values from the lists.
-
-`apply_css_to_select(select)` - Defines a set of css styles that can be applied to any listbox widget.
-
-`update_sconnects(b):` -
-This function is executed on the onclick event of the ‘Assign’ button. The notebook will first try to get the value from the 'Quick IP Scoring' textbox ignoring the selections from the listboxes; in case the textbox is empty, it will then
- get the selected values from each of the listboxes to look them up in the `flow_scores.csv` file.
-A binary search on the file is then performed:
-- The value in the 'Quick IP Scoring' textbox, will be compared against the `ip_src` and `ip_dst` columns; if either column is a match, the `sev` column will be updated with the value selected from the radiobutton list.
-- The column `srcIP` will be compared against the 'Source IP' selected value.
-- The column `dstIP` will be compared against the 'Dest IP' selected value.
-- The column `sport` will be compared against the 'Src Port' selected value.
-- The column `dport` will be compared against the 'Dst Port' selected value.
-
-Every row will be then appended to the `flow_scores.csv.tmp` file, which will replace the original `flow_scores.csv` at the end of the process.
-The scored rows will also be appended to the `flow_scores_fb.csv` file, which will later be used for the ML feedback.
-
-`set_rules():` - Predefined function where the user can define custom rules to be initally applied to the dataset. By default this function is commented out.
-
-`create_feedback_file(scored_rows):` - Appends the updated rows to the _flow_scores_fb.csv_ everytime a connection is scored. This file is used as feedback for the ML process.
-
-`apply_rules(rops,rvals,risk):` - This function applies the rules defined by `set_rules()` and updates the `flow_scores.csv` file following a similar process to the `update_sconnects()` function. By default this function is commented out.
-
-`attack_heuristics():` - This function is executed at the start, and loads the data from `flow_scores.csv` into a pandas dataframe grouped by `srcIp` column,
-to then print only those IP's that connect to more than 20 other different IP's. By default this function is commented out.
-
-`savesort(b):` - This event is triggered by the 'Save' button, and executes javascript functions to refresh the data on all the panels in Suspicious Connects.
-This function also reorders the _flow_scores.csv_ file by moving all scored connections to the end of the file and sorting the remaining connections by `lda_score` column.
-Finally, removes the widget panel and reloads it again to update the results, removing the need of a manual refresh, and calls the `ml_feedback():` function.
-
-`ml_feedback():` - A shell script is executed, transferring thru secure copy the _flow_scores_fb.csv_ file into ML Master node, where the destination path is defined at the spot.conf file.
-
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb b/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb
deleted file mode 100644
index e916471..0000000
--- a/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb
+++ /dev/null
@@ -1,407 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Initialize workflow."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import struct, socket\n",
- "import shutil\n",
- "import numpy as np\n",
- "import pandas as pd\n",
- "import linecache, bisect\n",
- "import csv, json\n",
- "import operator\n",
- "import os, time, subprocess \n",
- "from collections import OrderedDict\n",
- "\n",
- "try:\n",
- " import ipywidgets as widgets # For jupyter/ipython >= 1.4\n",
- "except ImportError:\n",
- " from IPython.html import widgets\n",
- "\n",
- "from IPython.display import display, Javascript, clear_output\n",
- "\n",
- "path = os.getcwd().split(\"/\") \n",
- "date = path[len(path)-1] \n",
- "dsource = path[len(path)-2] \n",
- "dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
- "cpath = '/'.join(['context' if var == 'ipynb' else var for var in path][:len(path)-2]) + '/'\n",
- "opath = '/'.join(['oa' if var == 'ipynb' else var for var in path][:len(path)-1]) + '/'\n",
- "sconnect = dpath + 'flow_scores.csv' \n",
- "sconnectbu = dpath + 'flow_scores_bu.csv'\n",
- "score_fbk = dpath + 'flow_scores_fb.csv'\n",
- "tmpconnect = sconnect +'.tmp'\n",
- "stemp = sconnect + '.new'\n",
- "file_schemas = opath + dsource + '_conf.json'\n",
- "#gets feedback columns from config file\n",
- "feedback_cols = json.loads(open (file_schemas).read(),object_pairs_hook=OrderedDict)['flow_feedback_fields']\n",
- "coff = 250;\n",
- "nwloc = cpath + 'networkcontext.csv' \n",
- "srcdict,srclist = {},[]\n",
- "dstdict,dstlist = {},[]\n",
- "sportdict,sportlist = {},[]\n",
- "dportdict,dportlist = {},[]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "def apply_css_to_select(select):\n",
- " select._css = (\n",
- " (None, 'height', '90%'),\n",
- " (None, 'width', '90%'),\n",
- " ('select', 'overflow-x', 'auto'),\n",
- " ('select', 'width', '100%'),\n",
- " ('select', 'margin', 0)\n",
- " )\n",
- "\n",
- "\n",
- "#load LDA scores#update lda doc with scores - either one edge,# or ALL that contain x IP and y port\n",
- "def displaythis() :\n",
- " # build dict of IP addresses\n",
- " #sev,score, tstart,srcIP,dstIP,sport,dport,proto,ipkt,ibyt\n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- " srcdict,srclist = {},[]\n",
- " dstdict,dstlist = {},[]\n",
- " sportdict,sportlist = {},[]\n",
- " dportdict,dportlist = {},[]\n",
- " srclist.append('- Select -')\n",
- " dstlist.append('- Select -')\n",
- " sportlist.append('- Select -')\n",
- " dportlist.append('- Select -')\n",
- " set_rules()\n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f,delimiter=',') \n",
- " rowct = 1\n",
- " for row in reader:\n",
- " if row['srcIP'] not in srcdict and row['sev'] == '0':\n",
- " srclist.append(row['srcIP'])\n",
- " srcdict[row['srcIP']] = struct.unpack(\"!L\", socket.inet_aton(row['srcIP']))[0]\n",
- " if row['dstIP'] not in dstdict and row['sev'] == '0':\n",
- " dstlist.append(row['dstIP'])\n",
- " dstdict[row['dstIP']] = struct.unpack(\"!L\", socket.inet_aton(row['dstIP']))[0]\n",
- " if row['sport'] not in sportdict and row['sev'] == '0':\n",
- " sportlist.append(row['sport'])\n",
- " sportdict[row['sport']] = row['sport']\n",
- " if row['dport'] not in dportdict and row['sev'] == '0':\n",
- " dportlist.append(row['dport'])\n",
- " dportdict[row['dport']] = row['dport']\n",
- " if rowct == coff:\n",
- " break;\n",
- " rowct += 1\n",
- " \n",
- " # Source IP box\n",
- " scrIpLalbel = widgets.HTML(value=\"Source IP:\", height='10%', width='100%')\n",
- " srcselect = widgets.Select(options=srclist, height='90%')\n",
- " apply_css_to_select(srcselect)\n",
- " srcIpBox = widgets.Box(width='25%', height='100%')\n",
- " srcIpBox.children = (scrIpLalbel, srcselect)\n",
- " \n",
- " # Destination IP box\n",
- " dstIpLalbel = widgets.HTML(value=\"Dest IP:\", height='10%', width='100%')\n",
- " dstselect = widgets.Select(options=dstlist, height='90%')\n",
- " apply_css_to_select(dstselect)\n",
- " dstIpBox = widgets.Box(width='25%', height='100%')\n",
- " dstIpBox.children = (dstIpLalbel, dstselect)\n",
- " \n",
- " # Source Port box\n",
- " scrPortLalbel = widgets.HTML(value=\"Src Port:\", height='10%', width='100%')\n",
- " sportselect = widgets.Select(options=sportlist, height='90%')\n",
- " apply_css_to_select(sportselect)\n",
- " srcPortBox = widgets.Box(width='20%', height='100%')\n",
- " srcPortBox.children = (scrPortLalbel, sportselect)\n",
- " \n",
- " # Destionation Port box\n",
- " dstPortLalbel = widgets.HTML(value=\"Dst Port:\", height='10%', width='100%')\n",
- " dportselect = widgets.Select(options=dportlist,height='90%')\n",
- " apply_css_to_select(dportselect)\n",
- " dstPortBox = widgets.Box(width='20%', height='100%')\n",
- " dstPortBox.children = (dstPortLalbel, dportselect)\n",
- " \n",
- " # Quick Search and Actions Box\n",
- " emptyLalbel = widgets.HTML(value=\" \")\n",
- " srctext = widgets.Text(value='', width='100%', placeholder='Quick IP scoring')\n",
- " srctext._css = (\n",
- " (None, 'width', '100%'),\n",
- " )\n",
- " ratingbut = widgets.RadioButtons(description='Rating:',options=['1', '2', '3'], width='100%')\n",
- " assignbut = widgets.Button(description='Score', width='45%')\n",
- " assignbut.button_style = 'primary'\n",
- " updatebut = widgets.Button(description='Save', width='45%')\n",
- " updatebut.button_style = 'primary'\n",
- " updatebut._css = (\n",
- " (None, 'margin-left', '10%'),\n",
- " )\n",
- " actionsBox = widgets.Box(width='20%', height='100%')\n",
- " actionsBox.children = (emptyLalbel, srctext,ratingbut,assignbut,updatebut)\n",
- " \n",
- " # Container Box\n",
- " bigBox = widgets.HBox(width='90%', height=250)\n",
- " bigBox.children = (srcIpBox, dstIpBox, srcPortBox, dstPortBox, actionsBox)\n",
- " \n",
- " display(bigBox)\n",
- " \n",
- " def update_sconnects(b): \n",
- " clear_output()\n",
- " time.sleep(.25)\n",
- " dvals,svals = [], [] \n",
- " scored_threats =[]\n",
- " #define logic based on combo of input\n",
- " #Gets input values\n",
- " if srctext.value != '':\n",
- " svals = [srctext.value,dstselect.value,sportselect.value,dportselect.value]\n",
- " dvals = [srcselect.value,srctext.value,sportselect.value,dportselect.value] \n",
- " else:\n",
- " svals = [srcselect.value,dstselect.value,sportselect.value,dportselect.value]\n",
- " dvals = [] \n",
- " risk = ratingbut.value \n",
- " shash, dhash = 0, 0\n",
- " fhash = ['srcIP','dstIP','sport','dport'] \n",
- " \n",
- " for k in xrange(len(svals)):\n",
- " if svals[k] == '- Select -': svals[k] = ''\n",
- " if svals[k] != '': shash += 2**k \n",
- " if len(dvals) > 0:\n",
- " if dvals[k] == '- Select -': dvals[k] = ''\n",
- " if dvals[k] != '': dhash += 2**k \n",
- " \n",
- " rowct = 0\n",
- " threat = []\n",
- " if shash > 0 or dhash > 0: \n",
- " with open(tmpconnect,'w') as g:\n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.DictReader(f,delimiter=',')\n",
- " riter = csv.DictWriter(g,delimiter=',', fieldnames=reader.fieldnames)\n",
- " riter.writeheader()\n",
- " \n",
- " for row in reader: \n",
- " result, resultd = 0,0\n",
- " for n in xrange(0,len(svals)):\n",
- " if (2**n & shash) > 0: \n",
- " if row[fhash[n]] == svals[n]:\n",
- " result += 2**n \n",
- " if result == shash:\n",
- " row['sev'] = risk \n",
- " scored_threats.append({col:row[col] for col in feedback_cols.keys()})\n",
- " rowct += 1\n",
- "\n",
- " if len(dvals) > 0:\n",
- " for n in xrange(0,len(dvals)):\n",
- " if (2**n & dhash) > 0: \n",
- " if row[fhash[n]] == dvals[n]:\n",
- " resultd += 2**n \n",
- " if resultd == dhash:\n",
- " row['sev'] = risk\n",
- " scored_threats.append({col:row[col] for col in feedback_cols.keys()})\n",
- " rowct += 1\n",
- " \n",
- " riter.writerow(row) \n",
- "\n",
- " create_feedback_file(scored_threats)\n",
- " shutil.copyfile(tmpconnect,sconnect)\n",
- " \n",
- " print \"{0} matching connections scored\".format(rowct)\n",
- " \n",
- " \n",
- " def savesort(b):\n",
- " clear_output()\n",
- " with open(stemp,'w') as g:\n",
- " reader = csv.DictReader(open(sconnect), delimiter=\",\")\n",
- " riter = csv.DictWriter(g,fieldnames=reader.fieldnames, delimiter=',')\n",
- " srtlist = sorted(reader, key=lambda x: (int(x[\"sev\"]), float(x[\"score\"])))\n",
- " riter.writeheader()\n",
- " riter.writerows(srtlist)\n",
- " \n",
- " shutil.copyfile(stemp,sconnect)\n",
- " print \"Suspicious connects successfully updated\" \n",
- " display(Javascript('reloadParentData();')) \n",
- " bigBox.close()\n",
- " # Rebuild widgets form\n",
- " displaythis()\n",
- " ml_feedback()\n",
- " assignbut.on_click(update_sconnects)\n",
- " updatebut.on_click(savesort)\n",
- "\n",
- " \n",
- "def create_feedback_file(scored_rows):\n",
- "# #works on the feedback tab-separated file\n",
- " if not os.path.exists(score_fbk): \n",
- " with open(score_fbk, 'w') as feedback:\n",
- " wr = csv.DictWriter(feedback, fieldnames=feedback_cols, delimiter='\\t', quoting=csv.QUOTE_NONE) \n",
- " wr.writeheader()\n",
- "\n",
- " wr = csv.DictWriter(open(score_fbk, 'a'), delimiter='\\t', fieldnames=feedback_cols, quoting=csv.QUOTE_NONE)\n",
- " for row in scored_rows:\n",
- " wr.writerow(row)\n",
- "\n",
- "\n",
- "def set_rules():\n",
- " rops = ['leq','leq','leq','leq','leq','leq']\n",
- " rvals = ['','','',1024,'',54]\n",
- " risk = 2\n",
- " apply_rules(rops,rvals,risk)\n",
- " rops = ['leq','leq','leq','leq','eq','eq']\n",
- " rvals = ['','','',1024,3,152]\n",
- " risk = 2\n",
- " apply_rules(rops,rvals,risk)\n",
- " rops = ['leq','leq','leq','leq','eq','eq']\n",
- " rvals = ['','','',1024,2,104]\n",
- " risk = 2\n",
- " rops = ['leq','leq','eq','leq','leq','leq']\n",
- " rvals = ['','',0,1023,'','']\n",
- " risk = 2\n",
- " apply_rules(rops,rvals,risk)\n",
- "\n",
- " \n",
- " \n",
- "def apply_rules(rops,rvals,risk):\n",
- " #define logic based on combo of input\n",
- " rhash = 0\n",
- " rfhash = ['srcIP','dstIP','sport','dport', 'ipkt', 'ibyt']\n",
- " scored_threats=[]\n",
- " \n",
- " for k in xrange(len(rvals)):\n",
- " if rvals[k] != '': \n",
- " rhash += 2**k\n",
- " \n",
- " with open(sconnect, 'r') as f:\n",
- " with open(tmpconnect,'w') as g:\n",
- " reader = csv.DictReader(f,delimiter=',')\n",
- " riter = csv.DictWriter(g,fieldnames=reader.fieldnames,delimiter=',')\n",
- " riter.writeheader()\n",
- " for row in reader: \n",
- " result = 0\n",
- " for n in xrange(0,len(rvals)):\n",
- " if (2**n & rhash) > 0:\n",
- " if rops[n] == 'leq':\n",
- " if int(row[rfhash[n]]) <= int(rvals[n]):\n",
- " result += 2**n \n",
- " if rops[n] == 'eq':\n",
- " if int(row[rfhash[n]]) == int(rvals[n]):\n",
- " result += 2**n \n",
- " if result == rhash:\n",
- " row['sev'] = risk\n",
- " scored_threats.append({col:row[col] for col in feedback_cols.keys()})\n",
- " riter.writerow(row) \n",
- " \n",
- " create_feedback_file(scored_threats)\n",
- " shutil.copyfile(tmpconnect,sconnect)\n",
- " \n",
- " \n",
- "def attack_heuristics():\n",
- " with open(sconnect, 'rb') as f:\n",
- " reader = csv.DictReader(f,delimiter=',') \n",
- " reader.next();\n",
- " rowct = 1\n",
- " for row in reader:\n",
- " if row['srcIP'] not in srcdict:\n",
- " srcdict[row['srcIP']] = row['srcIP']\n",
- " if row['dstIP'] not in dstdict:\n",
- " dstdict[row['dstIP']] = row['dstIP']\n",
- " if row['sport'] not in sportdict:\n",
- " sportdict[row['sport']] = row['sport']\n",
- " if row['dport'] not in dportdict:\n",
- " dportdict[row['dport']] = row['dport']\n",
- "\n",
- " df = pd.read_csv(sconnect) \n",
- " gb = df.groupby([u'srcIP']) \n",
- " \n",
- " for srcip in srcdict:\n",
- " try:\n",
- " if len(gb.get_group(srcip)) > 20:\n",
- " print srcip,'connects:',len(gb.get_group(srcip))\n",
- " except:\n",
- " print \"Key Error for ip: \" + srcip\n",
- " \n",
- " \n",
- "def ml_feedback():\n",
- " dst_name = os.path.basename(sconnect)\n",
- " str_fb=\"DSOURCE={0} &&\\\n",
- " FDATE={1} &&\\\n",
- " source /etc/spot.conf &&\\\n",
- " usr=$(echo $LUSER | cut -f3 -d'/') &&\\\n",
- " mlnode=$MLNODE &&\\\n",
- " lpath=$LPATH &&\\\n",
- " scp {2} $usr@$mlnode:$lpath/{3}\".format(dsource,date,score_fbk,dst_name) \n",
- " \n",
- " subprocess.call(str_fb, shell=True)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Run attack heuristics."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# set_rules()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# attack_heuristics()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "displaythis()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# !cp $sconnectbu $sconnect"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2.0
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.5"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/flow/ipynb_templates/ThreatInvestigation.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/ipynb_templates/ThreatInvestigation.md b/spot-oa/oa/flow/ipynb_templates/ThreatInvestigation.md
index fdbcad0..c9c7e2a 100644
--- a/spot-oa/oa/flow/ipynb_templates/ThreatInvestigation.md
+++ b/spot-oa/oa/flow/ipynb_templates/ThreatInvestigation.md
@@ -1,12 +1,13 @@
-#Flow Threat Investigation Notebook
+# Flow Threat Investigation Notebook
-###Dependencies
+### Dependencies
- [iPython == 3.2.1](https://ipython.org/ipython-doc/3/index.html)
- [Python 2.7.6](https://www.python.org/download/releases/2.7.6/)
- [ipywidgets 5.1.1](https://ipywidgets.readthedocs.io/en/latest/user_install.html#with-pip)
The following python modules will have to be imported for the notebook to work correctly:
+ import datetime
import struct, socket
import numpy as np
import linecache, bisect
@@ -14,176 +15,68 @@ The following python modules will have to be imported for the notebook to work c
import operator
import json
import os
+ import pandas as pd
import ipywidgets as widgets # For jupyter/ipython >= 1.4
from IPython.html import widgets
from IPython.display import display, Javascript, clear_output
-##Pre-requisites
+## Pre-requisites
- Execution of the spot-oa process for Flow
+- Correct installation of the UI [Read more](/ui/INSTALL.md)
- Score a set connections at the Edge Investigation Notebook
- Correct setup the spot.conf file. [Read more](/wiki/Edit%20Solution%20Configuration)
- Include a comma separated network context file. **Optional** [Schema](/spot-oa/oa/components/README.md#network-context-nc)
-- Include a geolocation database file. [Schema](/spot-oa/oa/components/README.md#geoloc)
+- Include a geolocation database file. **Optional** [Schema](/spot-oa/oa/components/README.md#geoloc)
-##Additional Configuration
+## Additional Configuration inside the notebook
`top_results` - This value defines the number of rows that will be displayed onscreen after the expanded search.
This also affects the number of IPs that will appear in the Timeline chart.
-##Data source
-The whole process in this notebook depends entirely on the existence of the scored _flow_scores.csv_ file, which is generated at the OA process, and scored at the Edge Investigation Notebook.
+## Data source
+Data should exists in the following tables:
+ *flow*
+ *flow_threat_investigation*
+
**Input files**
All these paths should be relative to the main OA path.
Schema for these files can be found here:
-
-[flow_scores.csv](/spot-oa/oa/flow)
+
[iploc.csv](/spot-oa/oa/components/README.md#geoloc)
[networkcontext_1.csv](/spot-oa/oa/components/README.md#network-context-nc)
-
- data/flow/<date>/flow_scores.csv
+
context/iploc.csv
context/networkcontext_1.csv
-**Output files**
-- threats.csv : Pipe separated file containing the comments saved by the user. This file is updated every time the user adds comments for a new threat.
-
- Schema with zero-indexed columns:
-
- 0.ip: string
- 1.title: string
- 2.description: string
-
-- sbdet-\<ip>.tsv : Tab separated file, this file lists all the client IP's that connected to the IP under investigation, including:
-the duration of the connection, response code and exact date and time of each the connection.
- Schema with zero-indexed columns:
-
- 0.tstart: string
- 1.tend: string
- 2.srcip : string
- 3.dstip : string
- 4.proto : string
- 5.sport : string
- 6.dport : string
- 7.pkts : string
- 8.bytes : string
-
-
-- globe-\<ip>.json : Json file including the geolocation of all the suspicious connections related to the IP under investigation.
- Schema:
-
- {
- "destips": [{
- "geometry": {
- "type": "Point",
- "coordinates": <[Lat, Long] values from the geolocation database>
- },
- "type": "Feature",
- "properties": {
- "ip": "<dst IP>",
- "type": <1 for Inbound, 2 for Outbound, 3 for Two way>,
- "location": "<Host name provided by the geolocation db>"
- },
- ......
- }
- }],
- "type": "FeatureCollection",
- "sourceips": [{
- "geometry": {
- "type": "Point",
- "coordinates": <[Lat, Long] values from the geolocation database>
- },
- "type": "Feature",
- "properties": {
- "ip": "<src ip>",
- "type": <1 for Inbound, 2 for Outbound, 3 for Two way>,
- "location": "<Host name provided by the geolocation db>"
- },
- ......
- }]
- }
-
-
-- stats-\<ip>.json: Json file containing the count of connections of each kind made to/from the suspicious IP.
- Schema:
-
- {
- "size": <total of connections>,
- "name": "<Name of suspicious IP, according to the network context",
- "children": [{
- "size": <Total number of Inbound connections>,
- "name": "Inbound Only",
- "children":
- [{
- "name": "<Context name>",
- "size": <Number of connections>
- }, ...
- ]
- },
- {"size": <Total number of Outbound connections>,
- "name": "Outbound Only",
- "children":
- [{
- "name": "<Context name>",
- "size": <Number of connections>
- }, ...
- ]
- },
- {"size": <Total number of Two way connections>,
- "name": "two way",
- "children":
- [{
- "name": "<Context name>",
- "size": <Number of connections>
- }, ...
- ]
- }]
- }
-
-
- - threat-dendro-\<ip>.json : Json file including the breakdown of the connections performed by the suspicious IP.
-
- Schema:
-
- {"time": "date in YYYYMMDD format>",
- "name": "<suspicious IP>",
- "children": [{
- "impact": 0,
- "name": "<Type of connections>",
- "children": [
- <Individual connections named after the network context>
- ]
- }]
- }
+**Output**
+The following tables will be populated after the threat investigation process:
+ *flow_storyboard*
+ *flow_timeline*
-**HDFS tables consumed**
-
- flow
-
+The following files will be created and stored in HDFS.
+
+ globe-\<ip>.json
+ stats-\<ip>.json:
+ threat-dendro-\<ip>.json
-##FUNCTIONS
+## FUNCTIONS
**Widget configuration**
This is not a function, but more like global code to set up styles and widgets to format the output of the notebook.
-`start_investigation():` - This function cleans the notebook from previous executions, then loops through the _flow_scores.csv_ file to get the 'srcIp' and 'dstIP' values from connections scored as high risk (sev = 1), ignoring IPs
-already saved in the _threats.csv_ file.
+`start_investigation():` - This function cleans the notebook from previous executions, and calls the *threats* query to get the source and destination IP's previously scored as high risk.
`display_controls(threat_list):` - This function will display the ipython widgets with the listbox of high risk IP's and the "Search" button.
-`search_ip()` - This function is triggered by the onclick event of the "Search" button after selecting an IP from the listbox. This will perform a query to the _flow_ table to find all connections involving the selected IP.
- The results are stored in the _ir-\<ip>.tsv_ file. If the file is not empty, this will immediately execute the following functions:
- - get_in_out_and_twoway_conns()
- - add_geospatial_info()
- - add_network_context()
- - display_threat_box()
+`search_ip()` - This function is triggered by the onclick event of the "Search" button after selecting an IP from the listbox. This calls the graphql *threat / details* query to find additional connections involving the selected IP.
-`get_in_out_and_twoway_conns():` - With the data from the _ir-\<ip>.tsv_ file, this function will loop through each connection and store it into one of three dictionaries:
+`get_in_out_and_twoway_conns():` - With the data from the previous method, this function will loop through each connection and store it into one of three dictionaries:
- All unique ‘inbound’ connected IP's (Where the internal sought IP appears only as destination, or the opposite if the IP is external)
- All unique ‘outbound’ connected IP's (Where the internal sought IP appears only as source, or the opposite if the IP is external)
- All unique ‘two way’ connected IP's (Where the sought IP appears as both source and destination)
@@ -197,18 +90,7 @@ To aid on the analysis, this function displays four html tables each containing
`display_threat_box(ip):` - Displays the widgets for "Title", "Comments" textboxes and the "Save" button on the notebook, so the user can add comments related to the threat and save them to continue with the analysis.
-`add_network_context()` - This function depends on the existence of the _networkcontext\_1.csv_ file, otherwise this step will be skipped.
-This function will loop through all dictionaries updating each IP with its context depending on the ranges defined in the networkcontext.
-
-`add_geospatial_info()` - This function depends on the existence of the _iploc.csv_ file. This will read through the dictionaries created, looking for every IP and updating its geolocation data according to the iploc database. If the iploc file doesn't exist, this function will be skipped.
-
-`save_threat_summary()` - This function is triggered by the onclick event of the "Save" button. Removes the widgets and cleans the notebook from previous executions, removes the selected value from the listbox widget and
- executes each of the following functions to create the data source files for the storyboard:
-- generate_attack_map_file()
-- generate_stats()
-- generate_dendro()
-- details_inbound()
-- add_threat()
+`save_threat_summary()` - This function is triggered by the onclick event of the "Save" button. Removes the widgets and cleans the notebook from previous executions, removes the selected value from the listbox widget and executes the *createStoryboard* mutation to save the data for the storyboard.
`display_results(cols, dataframe, top)` -
*cols*: List of columns to display from the dataframe
@@ -216,23 +98,8 @@ This function will loop through all dictionaries updating each IP with its conte
*top*: Number of top rows to display.
This function will create a formatted html table to display the provided dataframe.
-`generate_attack_map_file(ip, inbound, outbound, twoway): `- This function depends on the existence of the _iploc.csv_ file. Using the geospatial info previously added to the dictionaries, this function will create the _globe.json_ file. If the iploc file doesn't exist, this function will be skipped.
-
-`generate_stats(ip, inbound, outbound, twoway, threat_name):` - This function reads through each of the dictionaries to group the connections by type. The results are stored in the _stats-<ip>.json_ file.
-
-`generate_dendro(ip, inbound, outbound, twoway, date):` - This function groups the results from all three dictionaries into a json file, adding additionals level if the dictionaries include network context for each IP.
-The results are stored in the _threat-dendro-<ip>.json_ file.
-
-`details_inbound(anchor, inbound, outbond, twoway):` - This function executes a query to the _flow_ table looking for all additional information between the shought IP (threat) and the IP's in the 'top_n' dictionaries. The results will be stored in the _sbdet-<ip>.tsv_ file.
-
-`add_threat(ip,threat_title):`- Creates or updates the _threats.csv_ file, appending the IP and Title from the web form. This will serve as the menu for the Story Board.
-
`get_top_bytes(conns_dict, top):` - Orders a dictionary descendent by number of bytes, returns a dictionary with the top 'n' values. This dictionary will be printed onscreen, listing the most active connections first.
`get_top_conns(conns_dict, top):` - Orders a dictionary descendent by number of connections executed, returns a dictionary with the top 'n' values. This dictionary will be printed onscreen, listing the most active connections first.
-`file_is_empty(path):` - Performs a validation to check the file of a size to determine if it is empty.
-
`removeWidget(index):` - Javascript function that removes a specific widget from the notebook.
-
-`get_ctx_name(full_context): ` **Deprecated**
[05/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/lib/topojson.min.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/lib/topojson.min.js b/spot-oa/ui/flow/js/lib/topojson.min.js
index 752fcfb..c6aa3cd 100644
--- a/spot-oa/ui/flow/js/lib/topojson.min.js
+++ b/spot-oa/ui/flow/js/lib/topojson.min.js
@@ -1,3 +1 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
topojson=function(){function t(t,e){function n(e){var n=t.arcs[e],r=n[0],o=[0,0];return n.forEach(function(t){o[0]+=t[0],o[1]+=t[1]}),[r,o]}var r={},o={},a={};e.forEach(function(t){var e=n(t);(r[e[0]]||(r[e[0]]=[])).push(t),(r[e[1]]||(r[e[1]]=[])).push(~t)}),e.forEach(function(t){var e,r,i=n(t),c=i[0],s=i[1];if(e=a[c])if(delete a[e.end],e.push(t),e.end=s,r=o[s]){delete o[r.start];var u=r===e?e:e.concat(r);o[u.start=e.start]=a[u.end=r.end]=u}else if(r=a[s]){delete o[r.start],delete a[r.end];var u=e.concat(r.map(function(t){return~t}).reverse());o[u.start=e.start]=a[u.end=r.start]=u}else o[e.start]=a[e.end]=e;else if(e=o[s])if(delete o[e.start],e.unshift(t),e.start=c,r=a[c]){delete a[r.end];var f=r===e?e:r.concat(e);o[f.start=r.start]=a[f.end=e.end]=f}else if(r=o[c]){delete o[r.start],delete a[r.end];var f=r.map(function(t){return~t}).reverse().concat(e);o[f.start=r.end]=a[f.end=e.end]=f}else o[e.start]=a[e.end]=e;else if(e=o[c])if(delete o[e.start],e.unshift(~t),e.start=s,r=a[s]){del
ete a[r.end];var f=r===e?e:r.concat(e);o[f.start=r.start]=a[f.end=e.end]=f}else if(r=o[s]){delete o[r.start],delete a[r.end];var f=r.map(function(t){return~t}).reverse().concat(e);o[f.start=r.end]=a[f.end=e.end]=f}else o[e.start]=a[e.end]=e;else if(e=a[s])if(delete a[e.end],e.push(~t),e.end=c,r=a[c]){delete o[r.start];var u=r===e?e:e.concat(r);o[u.start=e.start]=a[u.end=r.end]=u}else if(r=o[c]){delete o[r.start],delete a[r.end];var u=e.concat(r.map(function(t){return~t}).reverse());o[u.start=e.start]=a[u.end=r.start]=u}else o[e.start]=a[e.end]=e;else e=[t],o[e.start=c]=a[e.end=s]=e});var i=[];for(var c in a)i.push(a[c]);return i}function e(e,r,o){function a(t){0>t&&(t=~t),(l[t]||(l[t]=[])).push(f)}function i(t){t.forEach(a)}function c(t){t.forEach(i)}function s(t){"GeometryCollection"===t.type?t.geometries.forEach(s):t.type in d&&(f=t,d[t.type](t.arcs))}var u=[];if(arguments.length>1){var f,l=[],d={LineString:i,MultiLineString:c,Polygon:c,MultiPolygon:function(t){t.forEach(c)}};s(r)
,l.forEach(3>arguments.length?function(t,e){u.push([e])}:function(t,e){o(t[0],t[t.length-1])&&u.push([e])})}else for(var p=0,h=e.arcs.length;h>p;++p)u.push([p]);return n(e,{type:"MultiLineString",arcs:t(e,u)})}function n(t,e){function n(t,e){e.length&&e.pop();for(var n,o=h[0>t?~t:t],a=0,i=o.length,c=0,s=0;i>a;++a)e.push([(c+=(n=o[a])[0])*f+d,(s+=n[1])*l+p]);0>t&&r(e,i)}function o(t){return[t[0]*f+d,t[1]*l+p]}function a(t){for(var e=[],r=0,o=t.length;o>r;++r)n(t[r],e);return 2>e.length&&e.push(e[0]),e}function i(t){for(var e=a(t);4>e.length;)e.push(e[0]);return e}function c(t){return t.map(i)}function s(t){var e=t.type,n="GeometryCollection"===e?{type:e,geometries:t.geometries.map(s)}:e in v?{type:e,coordinates:v[e](t)}:{type:null};return"id"in t&&(n.id=t.id),"properties"in t&&(n.properties=t.properties),n}var u=t.transform,f=u.scale[0],l=u.scale[1],d=u.translate[0],p=u.translate[1],h=t.arcs,v={Point:function(t){return o(t.coordinates)},MultiPoint:function(t){return t.coordinates.map
(o)},LineString:function(t){return a(t.arcs)},MultiLineString:function(t){return t.arcs.map(a)},Polygon:function(t){return c(t.arcs)},MultiPolygon:function(t){return t.arcs.map(c)}};return s(e)}function r(t,e){for(var n,r=t.length,o=r-e;--r>o;)n=t[o],t[o++]=t[r],t[r]=n}function o(t,e){for(var n=0,r=t.length;r>n;){var o=n+r>>>1;e>t[o]?n=o+1:r=o}return n}function a(t){function e(t,e){t.forEach(function(t){0>t&&(t=~t);var n=a[t]||(a[t]=[]);n[e]||(n.forEach(function(t){var n,r;r=o(n=i[e],t),n[r]!==t&&n.splice(r,0,t),r=o(n=i[t],e),n[r]!==e&&n.splice(r,0,e)}),n[e]=e)})}function n(t,n){t.forEach(function(t){e(t,n)})}function r(t,e){"GeometryCollection"===t.type?t.geometries.forEach(function(t){r(t,e)}):t.type in c&&c[t.type](t.arcs,e)}var a=[],i=t.map(function(){return[]}),c={LineString:e,MultiLineString:n,Polygon:n,MultiPolygon:function(t,e){t.forEach(function(t){n(t,e)})}};return t.forEach(r),i}return{version:"0.0.32",mesh:e,object:n,neighbors:a}}();
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/notebooks.js b/spot-oa/ui/flow/js/notebooks.js
new file mode 100644
index 0000000..5d9c60a
--- /dev/null
+++ b/spot-oa/ui/flow/js/notebooks.js
@@ -0,0 +1,72 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+ (
+ <form className="form-inline">
+ <div className="form-group">
+ <label htmlFor="dataDatePicker">Data Date:</label>
+ <div className="input-group input-group-xs">
+ <DateInput id="dataDatePicker" />
+ <div className="input-group-addon">
+ <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+ </div>
+ </div>
+ </div>
+ </form>
+ ),
+ document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+ <div id="spot-content">
+ <PanelRow maximized>
+ <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+ <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="flow/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+ </Panel>
+ </PanelRow>
+ </div>,
+ document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/ChordsDiagramStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/ChordsDiagramStore.js b/spot-oa/ui/flow/js/stores/ChordsDiagramStore.js
index a1275b8..35447fb 100755
--- a/spot-oa/ui/flow/js/stores/ChordsDiagramStore.js
+++ b/spot-oa/ui/flow/js/stores/ChordsDiagramStore.js
@@ -1,50 +1,78 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var d3 = require('d3');
-var assign = require('object-assign');
-
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var NetflowConstants = require('../constants/NetflowConstants');
-var RestStore = require('../../../js/stores/RestStore');
-
-var DATE_FILTER = 'date';
-var IP_FILTER = 'ip';
-
-var ChordsDiagramStore = assign(new RestStore(NetflowConstants.API_VISUAL_DETAILS), {
- _parser: d3.dsv('\t', 'text/plain'),
- errorMessages: {
- 404: 'No details available'
- },
- headers: {
- // TODO: Add Headers
- },
- setDate: function (date)
- {
- this.setRestFilter(DATE_FILTER, date.replace(/-/g, ''));
- },
- setIp: function (ip)
- {
- this.setRestFilter(IP_FILTER, ip.replace(/\./g, '_'));
- },
- getIp: function ()
- {
- return this.getRestFilter(IP_FILTER).replace(/_/g, '.');
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const IP_VAR = 'ip';
+const DATE_VAR = 'date';
+
+class ChordsDiagramStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date: SpotDateType!, $ip: SpotIpType!) {
+ flow {
+ ipDetails(date: $date, ip: $ip) {
+ srcip: srcIp
+ dstip: dstIp
+ ipkts: inPkts
+ ibytes: inBytes
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.flow.ipDetails;
+ }
+
+ setIp(ip) {
+ this.setVariable(IP_VAR, ip);
+ }
+
+ getIp() {
+ this.getVariable(IP_VAR);
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+}
+
+const cds = new ChordsDiagramStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- ChordsDiagramStore.setDate(action.date);
- break;
+ cds.setDate(action.date);
+ break;
case SpotConstants.SELECT_IP:
- ChordsDiagramStore.setIp(action.ip);
+ cds.setIp(action.ip);
+ break;
+ case SpotConstants.RELOAD_SUSPICIOUS:
+ cds.resetData();
break;
case SpotConstants.RELOAD_DETAILS_VISUAL:
- ChordsDiagramStore.reload();
+ cds.sendQuery();
break;
}
});
-module.exports = ChordsDiagramStore;
+module.exports = cds;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/CommentsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/CommentsStore.js b/spot-oa/ui/flow/js/stores/CommentsStore.js
index bdc5061..9b746be 100755
--- a/spot-oa/ui/flow/js/stores/CommentsStore.js
+++ b/spot-oa/ui/flow/js/stores/CommentsStore.js
@@ -1,33 +1,64 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-var d3 = require('d3');
-
-var NetflowConstants = require('../constants/NetflowConstants');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var RestStore = require('../../../js/stores/RestStore');
-
-var CommentsStore = assign(new RestStore(NetflowConstants.API_COMMENTS), {
- _parser: d3.dsv('|', 'text/plain'),
- errorMessages: {
- 404: 'Please choose a different date, no comments have been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(NetflowConstants.API_COMMENTS.replace('${date}', date.replace(/-/g, '')));
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+
+class CommentsStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType) {
+ flow {
+ threats {
+ comments(date: $date) {
+ ip
+ title
+ summary: text
+ }
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.flow.threats.comments;
+ }
+
+ setDate (date) {
+ this.setVariable(DATE_VAR, date);
+ }
+}
+
+const cs = new CommentsStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- CommentsStore.setDate(action.date);
+ cs.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- CommentsStore.reload();
+ cs.sendQuery();
break;
}
});
-module.exports = CommentsStore;
+module.exports = cs;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/DetailsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/DetailsStore.js b/spot-oa/ui/flow/js/stores/DetailsStore.js
index 78e37af..34e5ca9 100755
--- a/spot-oa/ui/flow/js/stores/DetailsStore.js
+++ b/spot-oa/ui/flow/js/stores/DetailsStore.js
@@ -1,77 +1,114 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
-var d3 = require('d3');
-var assign = require('object-assign');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var NetflowConstants = require('../constants/NetflowConstants');
-var RestStore = require('../../../js/stores/RestStore');
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
-var DATE_FILTER = 'date';
-var SRC_IP_FILTER = 'src_ip';
-var DST_IP_FILTER = 'dst_ip';
-var TIME_FILTER = 'time';
+const SRC_IP_VAR = 'srcIp';
+const DST_IP_VAR = 'dstIp';
+const TIME_VAR = 'tstart';
-var DetailsStore = assign(new RestStore(NetflowConstants.API_DETAILS), {
- _parser: d3.dsv('\t', 'text/plain'),
- errorMessages: {
- 404: 'No details available'
- },
- headers: {
- tstart: 'Time',
- srcip: 'Source IP',
- dstip: 'Destination IP',
- sport: 'Source Port',
- dport: 'Destination Port',
- proto: 'Protocol',
- flags: 'Flags',
- tos: 'Type Of Service',
- ibytes: 'Input Bytes',
- ipkts: 'Input Packets',
- obytes: 'Output Bytes',
- opkts: 'Output Packets',
- rip: 'Router IP',
- input: 'Input iface',
- output: 'Output iface'
- },
- ITERATOR: ['tstart', 'srcip', 'dstip', 'sport', 'dport', 'proto', 'flags', 'tos', 'ibytes', 'ipkts', 'obytes', 'opkts', 'rip', 'input', 'output'],
- setDate: function (date)
- {
- this.setRestFilter(DATE_FILTER, date.replace(/-/g, ''));
- },
- setSrcIp: function (ip)
- {
- this.setRestFilter(SRC_IP_FILTER, ip.replace(/\./g, '_'));
- },
- setDstIp: function (ip)
- {
- this.setRestFilter(DST_IP_FILTER, ip.replace(/\./g, '_'));
- },
- setTime: function (time)
- {
- var timeParts = time.split(' ')[1].split(':');
- this.setRestFilter(TIME_FILTER, timeParts[0] + '-' + timeParts[1]);
- }
-});
+class DetailStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+
+ this.headers = {
+ tstart: 'Time',
+ srcip: 'Source IP',
+ dstip: 'Destination IP',
+ sport: 'Source Port',
+ dport: 'Destination Port',
+ proto: 'Protocol',
+ flags: 'Flags',
+ tos: 'Type Of Service',
+ ibytes: 'Input Bytes',
+ ipkts: 'Input Packets',
+ obytes: 'Output Bytes',
+ opkts: 'Output Packets',
+ rip: 'Router IP',
+ input: 'Input iface',
+ output: 'Output iface'
+ };
+
+ this.ITERATOR = ['tstart', 'srcip', 'dstip', 'sport', 'dport', 'proto', 'flags', 'tos', 'ibytes', 'ipkts', 'obytes', 'opkts', 'rip', 'input', 'output'];
+ }
+
+ getQuery() {
+ return `
+ query($tstart: SpotDatetimeType!, $srcIp: SpotIpType!, $dstIp: SpotIpType!) {
+ flow {
+ edgeDetails(tstart: $tstart, srcIp: $srcIp, dstIp: $dstIp) {
+ tstart
+ srcip: srcIp
+ sport: srcPort
+ dstip: dstIp
+ dport: dstPort
+ proto: protocol
+ flags
+ tos
+ ipkts: inPkts
+ ibytes: inBytes
+ opkts: outPkts
+ obytes: outBytes
+ rip: routerIp
+ input: inIface
+ output: outIface
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.flow.edgeDetails;
+ }
+
+ setSrcIp(ip) {
+ this.setVariable(SRC_IP_VAR, ip);
+ }
+
+ setDstIp(ip) {
+ this.setVariable(DST_IP_VAR, ip);
+ }
+
+ setTime(time) {
+ this.setVariable(TIME_VAR, time);
+ }
+}
+
+const ds = new DetailStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
- case SpotConstants.UPDATE_DATE:
- DetailsStore.setDate(action.date);
- break;
case SpotConstants.SELECT_THREAT:
- DetailsStore.setSrcIp(action.threat.srcIP);
- DetailsStore.setDstIp(action.threat.dstIP);
- DetailsStore.setTime(action.threat.tstart);
+ ds.setSrcIp(action.threat.srcIP);
+ ds.setDstIp(action.threat.dstIP);
+ ds.setTime(action.threat.tstart);
break;
+ case SpotConstants.UPDATE_DATE:
case SpotConstants.RELOAD_SUSPICIOUS:
- DetailsStore.resetData();
+ ds.resetData();
break;
case SpotConstants.RELOAD_DETAILS:
- DetailsStore.reload();
+ ds.sendQuery();
break;
}
});
-module.exports = DetailsStore;
+module.exports = ds;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/GlobeViewStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/GlobeViewStore.js b/spot-oa/ui/flow/js/stores/GlobeViewStore.js
index f93a048..bcf3dfb 100755
--- a/spot-oa/ui/flow/js/stores/GlobeViewStore.js
+++ b/spot-oa/ui/flow/js/stores/GlobeViewStore.js
@@ -1,44 +1,89 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
-const assign = require('object-assign');
const d3 = require('d3');
const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
const NetflowConstants = require('../constants/NetflowConstants');
const SpotConstants = require('../../../js/constants/SpotConstants');
-const JsonStore = require('../../../js/stores/JsonStore');
-const IP_FILTER_NAME = 'ip';
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+const IP_VAR = 'ip';
let WORLD_DATA = null;
-const GlobeViewStore = assign(new JsonStore(NetflowConstants.API_GLOBE_VIEW), {
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(NetflowConstants.API_GLOBE_VIEW.replace('${date}', date.replace(/-/g, '')));
- },
- setIp: function (value)
- {
- this.setRestFilter(IP_FILTER_NAME, value);
- },
- getIp: function ()
- {
- return this.getRestFilter(IP_FILTER_NAME);
- },
- setData: function (data)
- {
- this._data = data;
-
- this.emitChangeData();
- },
+class GlobeViewStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!, $ip:SpotIpType!) {
+ flow {
+ threat {
+ geoLocalization(date: $date, ip: $ip) {
+ sourceips: srcIps {
+ geometry {
+ coordinates
+ }
+ properties {
+ ip
+ type
+ location
+ }
+ }
+ destips: dstIps {
+ geometry {
+ coordinates
+ }
+ properties {
+ ip
+ type
+ location
+ }
+ }
+ }
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.flow.threat.geoLocalization;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setIp(ip) {
+ this.setVariable(IP_VAR, ip);
+ }
+
+ getIp() {
+ return this.getVariable(IP_VAR);
+ }
+
getWorldData() {
return WORLD_DATA;
- },
- reload() {
+ }
+
+ sendQuery() {
if (WORLD_DATA instanceof Object) {
- Object.getPrototypeOf(GlobeViewStore).reload.call(this);
+ super.sendQuery();
}
else if (WORLD_DATA===true) {
// Do nothing, already loading world data
@@ -47,27 +92,29 @@ const GlobeViewStore = assign(new JsonStore(NetflowConstants.API_GLOBE_VIEW), {
WORLD_DATA = true; // Signal world data is loading
d3.json(NetflowConstants.API_WORLD_110M, (error, response) => {
WORLD_DATA = response;
- Object.getPrototypeOf(GlobeViewStore).reload.call(this);
+ super.sendQuery();
});
}
}
-});
+}
+
+const gvs = new GlobeViewStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- GlobeViewStore.setDate(action.date);
+ gvs.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- GlobeViewStore.resetData();
+ gvs.resetData();
break;
case SpotConstants.SELECT_COMMENT:
- GlobeViewStore.setIp(action.comment.ip);
- GlobeViewStore.reload();
+ gvs.setIp(action.comment.ip);
+ gvs.sendQuery();
break;
}
});
-module.exports = GlobeViewStore;
+module.exports = gvs;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/ImpactAnalysisStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/ImpactAnalysisStore.js b/spot-oa/ui/flow/js/stores/ImpactAnalysisStore.js
index b0b853c..256ec8a 100755
--- a/spot-oa/ui/flow/js/stores/ImpactAnalysisStore.js
+++ b/spot-oa/ui/flow/js/stores/ImpactAnalysisStore.js
@@ -1,47 +1,86 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-const assign = require('object-assign');
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-const FlowConstants = require('../constants/NetflowConstants');
const SpotConstants = require('../../../js/constants/SpotConstants');
-const JsonStore = require('../../../js/stores/JsonStore');
-
-const IP_FILTER_NAME = 'ip';
-
-const ImpactAnalysisStore = assign(new JsonStore(FlowConstants.API_IMPACT_ANALYSIS), {
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(FlowConstants.API_IMPACT_ANALYSIS.replace('${date}', date.replace(/-/g, '')));
- },
- setIp(value)
- {
- this.setRestFilter(IP_FILTER_NAME, value);
- },
- getIp()
- {
- this.getRestFilter(IP_FILTER_NAME);
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+const IP_VAR = 'ip';
+
+class ImpactAnalysisStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!, $ip:SpotIpType!) {
+ flow {
+ threat {
+ impactAnalysis(date: $date, ip: $ip) {
+ name
+ size
+ children {
+ name
+ size
+ children {
+ name
+ size
+ }
+ }
+ }
+ }
+ }
+ }
+ `;
}
-});
+
+ unboxData(data) {
+ return data.flow.threat.impactAnalysis;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setIp(ip) {
+ this.setVariable(IP_VAR, ip);
+ }
+
+ getIp() {
+ return this.getVariable(IP_VAR);
+ }
+}
+
+const ias = new ImpactAnalysisStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- ImpactAnalysisStore.setDate(action.date);
+ ias.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- ImpactAnalysisStore.resetData();
+ ias.resetData();
break;
case SpotConstants.SELECT_COMMENT:
- ImpactAnalysisStore.setIp(action.comment.ip);
- ImpactAnalysisStore.reload();
+ ias.setIp(action.comment.ip);
+ ias.sendQuery();
break;
}
});
-module.exports = ImpactAnalysisStore;
+module.exports = ias;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/IncidentProgressionStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/IncidentProgressionStore.js b/spot-oa/ui/flow/js/stores/IncidentProgressionStore.js
index 71a76c5..3f49039 100755
--- a/spot-oa/ui/flow/js/stores/IncidentProgressionStore.js
+++ b/spot-oa/ui/flow/js/stores/IncidentProgressionStore.js
@@ -1,46 +1,85 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-const assign = require('object-assign');
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
const SpotConstants = require('../../../js/constants/SpotConstants');
-const FlowConstants = require('../constants/NetflowConstants');
-const JsonStore = require('../../../js/stores/JsonStore');
-const IP_FILTER_NAME = 'ip';
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+const IP_VAR = 'ip';
+
+class IncidentProgressionStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!, $ip:SpotIpType!) {
+ flow {
+ threat {
+ incidentProgression(date: $date, ip: $ip) {
+ name
+ children {
+ name
+ children {
+ name
+ }
+ }
+ }
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.flow.threat.incidentProgression;
+ }
-const IncidentProgressionStore = assign(new JsonStore(FlowConstants.API_INCIDENT_PROGRESSION), {
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date)
+ setDate(date)
{
- this.setEndpoint(FlowConstants.API_INCIDENT_PROGRESSION.replace('${date}', date.replace(/-/g, '')));
- },
- setIp: function (value)
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setIp(ip)
{
- this.setRestFilter(IP_FILTER_NAME, value);
- },
- getIp: function ()
+ this.setVariable(IP_VAR, ip);
+ }
+
+ getIp()
{
- return this.getRestFilter(IP_FILTER_NAME);
+ return this.getVariable(IP_VAR);
}
-});
+}
+
+const ips = new IncidentProgressionStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- IncidentProgressionStore.setDate(action.date);
+ ips.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- IncidentProgressionStore.resetData();
+ ips.resetData();
break;
case SpotConstants.SELECT_COMMENT:
- IncidentProgressionStore.setIp(action.comment.ip);
- IncidentProgressionStore.reload();
+ ips.setIp(action.comment.ip);
+ ips.sendQuery();
break;
}
});
-module.exports = IncidentProgressionStore;
+module.exports = ips;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/NotebookStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/NotebookStore.js b/spot-oa/ui/flow/js/stores/NotebookStore.js
new file mode 100644
index 0000000..ba2f8f6
--- /dev/null
+++ b/spot-oa/ui/flow/js/stores/NotebookStore.js
@@ -0,0 +1,107 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SuspiciousStore = require('./SuspiciousStore');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const SCORED_ELEMENTS = 'input';
+const RESET_SCORED_ELEMENTS = 'date';
+
+class NotebookStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+ this.selectedQuery = 'scoreQuery';
+ this.query = {
+ scoreQuery: `
+ mutation($input:[NetflowScoreInputType!]!) {
+ flow{
+ score(input:$input)
+ {success}
+ }
+ }`,
+ resetQuery: `
+ mutation($date:SpotDateType!) {
+ flow{
+ resetScoredConnections(date:$date){
+ success
+ }
+ }
+ }`
+ };
+ this.completeClass = false;
+ }
+
+ getQuery() {
+ return this.query[this.selectedQuery];
+ }
+
+ unboxData(data) {
+ return data.flow.suspicious;
+ }
+
+ setScoredElements(scoredElements) {
+ this.selectedQuery = 'scoreQuery';
+ this.setVariable(SCORED_ELEMENTS, scoredElements);
+ this.unsetVariables(RESET_SCORED_ELEMENTS);
+ }
+
+ resetScoredElements(date) {
+ this.selectedQuery = 'resetQuery';
+ this.setVariable(RESET_SCORED_ELEMENTS, date);
+ this.unsetVariables(SCORED_ELEMENTS);
+ }
+
+ unsetVariables(variable) {
+ this.unsetVariable(variable);
+ }
+
+ reloadElements() {
+ SuspiciousStore.sendQuery();
+ }
+
+ changeCssCls() {
+ this.completeClass = !this.completeClass;
+ }
+
+
+}
+
+const ns = new NotebookStore();
+
+SpotDispatcher.register(function (action) {
+ switch (action.actionType) {
+ case SpotConstants.SAVE_SCORED_ELEMENTS:
+ ns.setScoredElements(action.scoredElems);
+ ns.sendQuery();
+ break;
+ case SpotConstants.RESET_SCORED_ELEMENTS:
+ ns.resetScoredElements(action.date);
+ ns.sendQuery();
+ break;
+ case SpotConstants.RELOAD_SUSPICIOUS:
+ ns.resetData();
+ break;
+ case SpotConstants.CHANGE_CSS_CLS:
+ ns.changeCssCls();
+ break;
+ }
+});
+
+module.exports = ns;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/SuspiciousStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/SuspiciousStore.js b/spot-oa/ui/flow/js/stores/SuspiciousStore.js
index c074f9f..e0fa11d 100755
--- a/spot-oa/ui/flow/js/stores/SuspiciousStore.js
+++ b/spot-oa/ui/flow/js/stores/SuspiciousStore.js
@@ -1,167 +1,187 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var NetflowConstants = require('../constants/NetflowConstants');
-var RestStore = require('../../../js/stores/RestStore');
-var SpotUtils = require('../../../js/utils/SpotUtils');
-
-var IP_FILTER = 'ip_dst';
-
-var CHANGE_FILTER_EVENT = 'change_filter';
-var HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
-var UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
-var SELECT_THREAT_EVENT = 'select_treath';
-
-var filter = '';
-var highlightedThread = null;
-var selectedThread = null;
-var unfilteredData = null;
-
-var SuspiciousStore = assign(new RestStore(NetflowConstants.API_SUSPICIOUS), {
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- headers: {
- rank: 'Rank',
- tstart: 'Time',
- srcIP: 'Source IP',
- srcIP_rep: ' ',
- dstIP: 'Destination IP',
- dstIP_rep: ' ',
- sport: 'Source Port',
- dport: 'Destination Port',
- proto: 'Protocol',
- ipkt: 'Input Packets',
- ibyt: 'Input Bytes',
- opkt: 'Output Packets',
- obyt: 'Output Bytes'
- },
- ITERATOR: ['rank', 'tstart', 'srcIP', 'srcIP_rep', 'dstIP', 'dstIP_rep', 'sport', 'dport', 'proto', 'ipkt', 'ibyt', 'opkt', 'obyt'],
- getData: function () {
- var state;
-
- if (!filter || !unfilteredData) {
- state = this._data;
- }
- else {
- state = assign(
- {},
- unfilteredData
- );
-
- if (unfilteredData.data) {
- state.data = unfilteredData.data.filter(function (item) {
- return item['srcIP'] == filter || item['dstIP'] == filter;
- });
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotConstants = require('../../../js/constants/SpotConstants');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const CHANGE_FILTER_EVENT = 'change_filter';
+const HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
+const UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
+const SELECT_THREAT_EVENT = 'select_treath';
+
+const DATE_VAR = 'date';
+const IP_VAR = 'ip';
+
+class SuspiciousStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+
+ this.highlightedThread = null;
+ this.selectedThread = null;
+
+ this.headers = {
+ rank: 'Rank',
+ tstart: 'Time',
+ srcIP: 'Source IP',
+ srcIP_rep: ' ',
+ dstIP: 'Destination IP',
+ dstIP_rep: ' ',
+ sport: 'Source Port',
+ dport: 'Destination Port',
+ proto: 'Protocol',
+ ipkt: 'Input Packets',
+ ibyt: 'Input Bytes',
+ opkt: 'Output Packets',
+ obyt: 'Output Bytes'
+ };
+
+ this.ITERATOR = ['rank', 'tstart', 'srcIP', 'srcIP_rep', 'dstIP', 'dstIP_rep', 'sport', 'dport', 'proto', 'ipkt', 'ibyt', 'opkt', 'obyt'];
+ }
+
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$ip:SpotIpType) {
+ flow {
+ suspicious(date: $date, ip:$ip) {
+ rank
+ tstart
+ srcIP: srcIp
+ sport: srcPort
+ srcIpInternal: srcIp_isInternal
+ srcIP_rep: srcIp_rep
+ srcGeo: srcIp_geoloc
+ srcDomain: srcIp_domain
+ dstIP: dstIp
+ dport: dstPort
+ destIpInternal: dstIp_isInternal
+ dstIP_rep: dstIp_rep
+ dstGeo: dstIp_geoloc
+ dstDomain: dstIp_domain
+ proto: protocol
+ ipkt: inPkts
+ ibyt: inBytes
+ opkt: outPkts
+ obyt: outBytes
+ score
+ }
+ }
}
- }
-
- if (state.data) {
- state.data = state.data.filter(function (item) {
- return item.sev == '0';
- });
-
- if (state.data.length > SpotConstants.MAX_SUSPICIOUS_ROWS) state.data = state.data.slice(0, SpotConstants.MAX_SUSPICIOUS_ROWS);
- }
-
- return state;
- },
- setData: function (data) {
- this._data = unfilteredData = data;
-
- this.emitChangeData();
- },
- setDate: function (date) {
- this.setEndpoint(NetflowConstants.API_SUSPICIOUS.replace('${date}', date.replace(/-/g, '')));
- },
- setFilter: function (newFilter) {
- filter = newFilter;
-
- this.emitChangeFilter();
- },
- getFilter: function () {
- return filter;
- },
- emitChangeFilter: function () {
- this.emit(CHANGE_FILTER_EVENT);
- },
- addChangeFilterListener: function (callback) {
- this.on(CHANGE_FILTER_EVENT, callback);
- },
- removeChangeFilterListener: function (callback) {
+ `;
+ }
+
+ unboxData(data) {
+ return data.flow.suspicious;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
+ setFilter(ip) {
+ this.setVariable(IP_VAR, ip || '');
+ this.notifyListeners(CHANGE_FILTER_EVENT);
+ }
+
+ getFilter() {
+ return this.getVariable(IP_VAR);
+ }
+
+ addChangeFilterListener(callback) {
+ this.addListener(CHANGE_FILTER_EVENT, callback);
+ }
+
+ removeChangeFilterListener(callback) {
this.removeListener(CHANGE_FILTER_EVENT, callback);
- },
- highlightThreat: function (threat) {
- highlightedThread = threat;
- this.emitHighlightThreat();
- },
- getHighlightedThreat: function () {
- return highlightedThread;
- },
- addThreatHighlightListener: function (callback) {
- this.on(HIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatHighlightListener: function (callback) {
+ }
+
+
+ highlightThreat (threat) {
+ this.highlightedThread = threat;
+ this.notifyListeners(HIGHLIGHT_THREAT_EVENT);
+ }
+
+ getHighlightedThreat() {
+ return this.highlightedThread;
+ }
+
+ addThreatHighlightListener(callback) {
+ this.addListener(HIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatHighlightListener(callback) {
this.removeListener(HIGHLIGHT_THREAT_EVENT, callback);
- },
- emitHighlightThreat: function () {
- this.emit(HIGHLIGHT_THREAT_EVENT);
- },
- unhighlightThreat: function () {
- highlightedThread = null;
- this.emitUnhighlightThreat();
- },
- addThreatUnhighlightListener: function (callback) {
- this.on(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatUnhighlightListener: function (callback) {
+ }
+
+ unhighlightThreat() {
+ this.highlightedThread = null;
+ this.notifyListeners(UNHIGHLIGHT_THREAT_EVENT);
+ }
+
+ addThreatUnhighlightListener(callback) {
+ this.addListener(UNHIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatUnhighlightListener(callback) {
this.removeListener(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- emitUnhighlightThreat: function () {
- this.emit(UNHIGHLIGHT_THREAT_EVENT);
- },
- selectThreat: function (threat) {
- selectedThread = threat;
- this.emitThreatSelect();
- },
- getSelectedThreat: function () {
- return selectedThread;
- },
- addThreatSelectListener: function (callback) {
- this.on(SELECT_THREAT_EVENT, callback);
- },
- removeThreatSelectListener: function (callback) {
+ }
+
+ selectThreat(threat) {
+ this.selectedThread = threat;
+ this.notifyListeners(SELECT_THREAT_EVENT);
+ }
+
+ getSelectedThreat() {
+ return this.selectedThread;
+ }
+
+ addThreatSelectListener(callback) {
+ this.addListener(SELECT_THREAT_EVENT, callback);
+ }
+
+ removeThreatSelectListener(callback) {
this.removeListener(SELECT_THREAT_EVENT, callback);
- },
- emitThreatSelect: function () {
- this.emit(SELECT_THREAT_EVENT);
}
-});
+}
+
+const ss = new SuspiciousStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_FILTER:
- SuspiciousStore.setFilter(action.filter);
+ ss.setFilter(action.filter);
break;
case SpotConstants.UPDATE_DATE:
- SuspiciousStore.setDate(action.date);
+ ss.setDate(action.date);
break;
case SpotConstants.RELOAD_SUSPICIOUS:
- SuspiciousStore.reload();
+ ss.sendQuery();
break;
case SpotConstants.HIGHLIGHT_THREAT:
- SuspiciousStore.highlightThreat(action.threat);
+ ss.highlightThreat(action.threat);
break;
case SpotConstants.UNHIGHLIGHT_THREAT:
- SuspiciousStore.unhighlightThreat();
+ ss.unhighlightThreat();
break;
case SpotConstants.SELECT_THREAT:
- SuspiciousStore.selectThreat(action.threat);
+ ss.selectThreat(action.threat);
break;
}
});
-module.exports = SuspiciousStore;
+module.exports = ss;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/stores/TimelineStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/stores/TimelineStore.js b/spot-oa/ui/flow/js/stores/TimelineStore.js
index a189d87..258845e 100755
--- a/spot-oa/ui/flow/js/stores/TimelineStore.js
+++ b/spot-oa/ui/flow/js/stores/TimelineStore.js
@@ -1,55 +1,86 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-const assign = require('object-assign');
-const d3 = require('d3');
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-const FlowConstants = require('../constants/NetflowConstants');
const SpotConstants = require('../../../js/constants/SpotConstants');
-const RestStore = require('../../../js/stores/RestStore');
-
-const IP_FILTER_NAME = 'ip';
-
-const TimelineStore = assign(new RestStore(FlowConstants.API_TIMELINE), {
- _parser: d3.tsv,
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(FlowConstants.API_TIMELINE.replace('${date}', date.replace(/-/g, '')));
- this._date = date;
- },
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+const IP_VAR = 'ip';
+
+class TimelineStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!, $ip:SpotIpType!) {
+ flow {
+ threat {
+ timeline(date: $date, ip: $ip) {
+ tstart
+ srcip: srcIp
+ sport: srcPort
+ dstip: dstIp
+ dport: dstPort
+ }
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.flow.threat.timeline;
+ }
+
+ setDate(date){
+ this.setVariable(DATE_VAR, date);
+ }
+
getDate() {
- return this._date;
- },
- setIp: function (value)
- {
- this.setRestFilter(IP_FILTER_NAME, value);
- },
- getIp: function ()
- {
- return this.getRestFilter(IP_FILTER_NAME);
+ return this.getVariable(DATE_VAR);
}
-});
+ setIp(ip) {
+ this.setVariable(IP_VAR, ip);
+ }
+
+ getIp () {
+ return this.getVariable(IP_VAR);
+ }
+}
+
+const ts = new TimelineStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- TimelineStore.setDate(action.date);
+ ts.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- TimelineStore.resetData();
+ ts.resetData();
break;
case SpotConstants.SELECT_COMMENT:
- TimelineStore.setIp(action.comment.ip);
+ ts.setIp(action.comment.ip);
- TimelineStore.reload();
+ ts.sendQuery();
break;
}
});
-module.exports = TimelineStore;
+module.exports = ts;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/storyboard.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/storyboard.js b/spot-oa/ui/flow/js/storyboard.js
index 6970553..fc31be9 100755
--- a/spot-oa/ui/flow/js/storyboard.js
+++ b/spot-oa/ui/flow/js/storyboard.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -11,6 +26,7 @@ const SpotUtils = require('../../js/utils/SpotUtils');
// Build and Render Toolbar
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
function resetFilterAndReload()
{
@@ -19,6 +35,11 @@ function resetFilterAndReload()
};
ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
(
<form className="form-inline">
<div className="form-group">
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/suspicious.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/suspicious.js b/spot-oa/ui/flow/js/suspicious.js
index cb6b0fa..b963355 100755
--- a/spot-oa/ui/flow/js/suspicious.js
+++ b/spot-oa/ui/flow/js/suspicious.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -11,6 +26,7 @@ const SpotUtils = require('../../js/utils/SpotUtils');
// Build and Render Toolbar
const FilterInput = require('./components/FilterInput.react');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
function resetFilterAndReload()
{
@@ -18,6 +34,15 @@ function resetFilterAndReload()
EdInActions.reloadSuspicious();
};
+function switchComponents () {
+ SpotUtils.switchDivs(SpotConstants.DETAILS_PANEL, SpotConstants.SCORING_PANEL);
+};
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
ReactDOM.render(
(
<form className="form-inline">
@@ -54,11 +79,9 @@ const Panel = require('../../js/components/Panel.react');
const SuspiciousPanel = require('./components/SuspiciousPanel.react');
const NetworkViewPanel = require('./components/NetworkViewPanel.react');
-const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+const ScoreNotebook = require('./components/ScoreNotebook.react');
const DetailsPanel = require('./components/DetailsPanel.react');
-const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure(SpotConstants.NOTEBOOK_PANEL);
-
ReactDOM.render(
<div id="spot-content">
<PanelRow>
@@ -69,14 +92,18 @@ ReactDOM.render(
<NetworkViewPanel />
</Panel>
</PanelRow>
- <PanelRow>
- <Panel title={ipynbClosure.getTitle()} container extraButtons={ipynbClosure.getButtons}>
- <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="flow/${date}/Edge_Investigation.ipynb" />
- </Panel>
- <Panel title={SpotConstants.DETAILS_PANEL} container expandable>
- <DetailsPanel title={SpotConstants.DETAILS_PANEL} />
- </Panel>
- </PanelRow>
+ <div className="sortable">
+ <PanelRow title={SpotConstants.SCORING_PANEL}>
+ <Panel title={SpotConstants.SCORING_PANEL} reloadable switchable onReload={EdInActions.reloadSuspicious} onSwitch={switchComponents} className="col-md-12">
+ <ScoreNotebook />
+ </Panel>
+ </PanelRow>
+ <PanelRow title={SpotConstants.DETAILS_PANEL}>
+ <Panel title={SpotConstants.DETAILS_PANEL} container switchable expandable onSwitch={switchComponents} className="col-md-12">
+ <DetailsPanel title={SpotConstants.DETAILS_PANEL} />
+ </Panel>
+ </PanelRow>
+ </div>
</div>,
document.getElementById('spot-content-wrapper')
);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/js/threat-investigation.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/threat-investigation.js b/spot-oa/ui/flow/js/threat-investigation.js
index 7bdfe20..f39f83c 100755
--- a/spot-oa/ui/flow/js/threat-investigation.js
+++ b/spot-oa/ui/flow/js/threat-investigation.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -7,6 +22,12 @@ const SpotActions = require('../../js/actions/SpotActions');
const SpotUtils = require('../../js/utils/SpotUtils');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
ReactDOM.render(
(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/package.json b/spot-oa/ui/flow/package.json
index 62f47ae..f5d1704 100755
--- a/spot-oa/ui/flow/package.json
+++ b/spot-oa/ui/flow/package.json
@@ -6,10 +6,12 @@
"watch-suspicious": "watchify js/suspicious.js -o js/suspicious.bundle.min.js -v -d",
"watch-threat-investigation": "watchify js/threat-investigation.js -o js/threat-investigation.bundle.min.js -v -d",
"watch-storyboard": "watchify js/storyboard.js -o js/storyboard.bundle.min.js -v -d",
- "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard",
+ "watch-notebooks": "watchify js/notebooks.js -o js/notebooks.bundle.min.js -v -d",
+ "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard && npm run build-notebooks",
"build-suspicious": "browserify js/suspicious.js | uglifyjs -cm > js/suspicious.bundle.min.js",
"build-threat-investigation": "browserify js/threat-investigation.js | uglifyjs -cm > js/threat-investigation.bundle.min.js",
- "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js"
+ "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js",
+ "build-notebooks": "browserify js/notebooks.js | uglifyjs -cm > js/notebooks.bundle.min.js"
},
"browserify": {
"transform": [
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/storyboard.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/storyboard.html b/spot-oa/ui/flow/storyboard.html
index 9f7fd1e..3b3c32e 100755
--- a/spot-oa/ui/flow/storyboard.html
+++ b/spot-oa/ui/flow/storyboard.html
@@ -21,6 +21,7 @@
<title>NetFlow :: Storyboard</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -70,62 +71,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../proxy/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../proxy/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../proxy/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/suspicious.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/suspicious.html b/spot-oa/ui/flow/suspicious.html
index f475f4b..5fea68e 100755
--- a/spot-oa/ui/flow/suspicious.html
+++ b/spot-oa/ui/flow/suspicious.html
@@ -21,6 +21,7 @@
<title>Netflow :: Suspicious</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -30,6 +31,9 @@
<link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
<!-- Spot styles -->
<link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
<!-- Favicon -->
<link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
@@ -74,62 +78,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../proxy/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../proxy/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../proxy/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
@@ -153,6 +102,7 @@
<script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
<script type="application/javascript" src="js/lib/ChordMapper.js"></script>
<script type="application/javascript" src="js/suspicious.bundle.min.js"></script>
</body>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/flow/threat-investigation.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/threat-investigation.html b/spot-oa/ui/flow/threat-investigation.html
index 97d70f3..0d51e7e 100755
--- a/spot-oa/ui/flow/threat-investigation.html
+++ b/spot-oa/ui/flow/threat-investigation.html
@@ -21,6 +21,7 @@
<title>Netflow :: Threat Investigation</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -68,62 +69,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../proxy/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../proxy/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../proxy/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/images/favicon/browserconfig.xml
----------------------------------------------------------------------
diff --git a/spot-oa/ui/images/favicon/browserconfig.xml b/spot-oa/ui/images/favicon/browserconfig.xml
index c554148..bdc4266 100644
--- a/spot-oa/ui/images/favicon/browserconfig.xml
+++ b/spot-oa/ui/images/favicon/browserconfig.xml
@@ -1,2 +1,16 @@
<?xml version="1.0" encoding="utf-8"?>
+<!--Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.-->
<browserconfig><msapplication><tile><square70x70logo src="/ms-icon-70x70.png"/><square150x150logo src="/ms-icon-150x150.png"/><square310x310logo src="/ms-icon-310x310.png"/><TileColor>#ffffff</TileColor></tile></msapplication></browserconfig>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/ingest-summary.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/ingest-summary.html b/spot-oa/ui/ingest-summary.html
deleted file mode 100755
index bbc9877..0000000
--- a/spot-oa/ui/ingest-summary.html
+++ /dev/null
@@ -1,183 +0,0 @@
-<!DOCTYPE html>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<html>
-<head>
- <title>Ingest Summary</title>
-
- <meta charset="UTF-8">
-
- <!--Bootstrap styles-->
- <link rel="stylesheet" type="text/css" href="css/bootstrap-spot.min.css" />
- <!--Bootstrap Date picker styles-->
- <link rel="stylesheet" type="text/css" href="node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
- <!-- Spot styles -->
- <link rel="stylesheet" type="text/css" href="css/main.css" />
- <!-- Favicon -->
- <link rel="apple-touch-icon" sizes="57x57" href="images/favicon/apple-icon-57x57.png">
- <link rel="apple-touch-icon" sizes="60x60" href="images/favicon/apple-icon-60x60.png"
- <link rel="apple-touch-icon" sizes="72x72" href="images/favicon/apple-icon-72x72.png">
- <link rel="apple-touch-icon" sizes="76x76" href="images/favicon/apple-icon-76x76.png">
- <link rel="apple-touch-icon" sizes="114x114" href="images/favicon/apple-icon-114x114.png">
- <link rel="apple-touch-icon" sizes="120x120" href="images/favicon/apple-icon-120x120.png">
- <link rel="apple-touch-icon" sizes="144x144" href="images/favicon/apple-icon-144x144.png">
- <link rel="apple-touch-icon" sizes="152x152" href="images/favicon/apple-icon-152x152.png">
- <link rel="apple-touch-icon" sizes="180x180" href="images/favicon/apple-icon-180x180.png">
- <link rel="icon" type="image/png" sizes="192x192" href="images/favicon/android-icon-192x192.png">
- <link rel="icon" type="image/png" sizes="32x32" href="images/favicon/favicon-32x32.png">
- <link rel="icon" type="image/png" sizes="96x96" href="images/favicon/favicon-96x96.png">
- <link rel="icon" type="image/png" sizes="16x16" href="images/favicon/favicon-16x16.png">
- <link rel="manifest" href="images/favicon/manifest.json">
- <meta name="msapplication-TileColor" content="#ffffff">
- <meta name="msapplication-TileImage" content="images/favicon/ms-icon-144x144.png">
- <meta name="theme-color" content="#ffffff">
- <style type="text/css">
- .spot-row {
- height: 100%;
- }
-
- .is-chart svg {
- width: 100%;
- height: 100%;
- }
-
- .is-chart svg .header text {
- text-anchor: middle;
- fill: #82837e;
- }
-
- .is-chart svg .header text tspan.bold {
- font-weight: bold;
- }
-
- .is-chart .axis {
- shape-rendering: crispEdges;
- }
-
- .is-chart .axis path, .is-chart .axis line {
- fill: none;
- }
-
- .is-chart .pipeline {
- pointer-events: all;
- }
-
- .is-chart .pipeline.zoom-in {
- cursor: zoom-in;
- }
-
- .is-chart .pipeline.zoom-out {
- cursor: zoom-out;
- }
-
- .is-chart .pipeline.e-resize {
- cursor: e-resize;
- }
- </style>
-</head>
-<body>
- <nav id="spot-nav" class="navbar navbar-default">
- <div class="container-fluid">
- <!-- App name and toggle get grouped for better mobile display -->
- <div class="navbar-header">
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <span class="navbar-brand">Apache Spot :: Ingest Summary</span>
- </div>
- <!-- Collect the nav links, forms, and other content for toggling -->
- <div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="flow/suspicious.html#date=${end-date}">Suspicious</a>
- </li>
- <li>
- <a data-href="flow/threat-investigation.html#date=${end-date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="flow/storyboard.html#date=${end-date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="dns/suspicious.html#date=${end-date}">Suspicious</a>
- </li>
- <li>
- <a data-href="dns/threat-investigation.html#date=${end-date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="dns/storyboard.html#date=${end-date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="proxy/suspicious.html#date=${end-date}">Suspicious</a>
- </li>
- <li>
- <a data-href="proxy/threat-investigation.html#date=${end-date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="proxy/storyboard.html#date=${end-date}">Storyboard</a>
- </li>
- </ul>
- </li>
- </ul>
- </div>
- <div id="search-box" class="row text-right">
- <!--Tools Buttons-->
- <div id="nav_form" class="col-md-12">
- <!-- Search form placeholder -->
- </div>
- </div> <!-- /Tools Buttons-->
- </div>
- </nav>
- <div id="spot-content-wrapper" class="container-fluid">
- <!-- Main Content Placeholder -->
- </div>
-
- <!-- SCRIPTS -->
- <script type="application/javascript" src="node_modules/jquery/dist/jquery.min.js"></script>
- <script type="application/javascript" src="node_modules/d3/d3.min.js"></script>
- <script type="application/javascript" src="node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
- <script type="application/javascript" src="node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
- <script type="application/javascript" src="node_modules/react/dist/react.min.js"></script>
- <script type="application/javascript" src="node_modules/react-dom/dist/react-dom.min.js"></script>
- <script type="application/javascript" src="js/ingest-summary.bundle.min.js"></script>
-</body>
-</html>
[13/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/README.md b/spot-oa/README.md
index ee9b85b..8c4732c 100644
--- a/spot-oa/README.md
+++ b/spot-oa/README.md
@@ -44,16 +44,16 @@ spot-oa is the root folder, below are more details about child folders:
* [EdgeNotebook](/spot-oa/oa/proxy/ipynb_templates/EdgeNotebook.md)
* [ThreatInvestigation](/spot-oa/oa/proxy/ipynb_templates/ThreatInvestigation.md)
-##**Views**
+## **Views**
-###Analyst View (suspicious.html)
+### Analyst View (suspicious.html)
The Analyst view provides a dashboard showing the top 250 suspicious connects.
-###**Threat Investigation (threat-investigation.html)**
+### **Threat Investigation (threat-investigation.html)**
HTML page container for the Threat Investigation Notebook
-###**Story Board (storyboard.html)**
+### **Story Board (storyboard.html)**
Executive Briefing dashboard
-###**Ingest Summary (ingest-summary.html)**
+### **Ingest Summary (ingest-summary.html)**
Interactive histogram dashboard that shows the amount of data captured in a certain period of time.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/__init__.py b/spot-oa/__init__.py
new file mode 100755
index 0000000..ecb1860
--- /dev/null
+++ b/spot-oa/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/__init__.py b/spot-oa/api/__init__.py
new file mode 100644
index 0000000..ecb1860
--- /dev/null
+++ b/spot-oa/api/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/README.md b/spot-oa/api/graphql/README.md
new file mode 100644
index 0000000..17fd7f6
--- /dev/null
+++ b/spot-oa/api/graphql/README.md
@@ -0,0 +1,13 @@
+# Apache Spot (incubating) - GraphQL API
+
+Provides an endpoint to send GraphQL queries to access and modify data.
+
+1. Install Spot OA. Follow this [guide](../../README.md#installation).
+2. Start Spot OA Web Server.
+ 1. From spot-oa dir, run ./runIpython.sh
+3. Apache Spot will deploy a GraphQL endpoint under /graphql URL
+
+## Development mode and GraphiQL UI
+
+When Spot OA Web server is started in [development mode](../../ui/README.md#developmentdebugging-process), a GraphiQL UI will be enabled at /graphql URL. Use
+this tool to play with our GraphQL API and to explore the entities we have made available.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/__init__.py b/spot-oa/api/graphql/__init__.py
new file mode 100644
index 0000000..481d353
--- /dev/null
+++ b/spot-oa/api/graphql/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from webapp import blueprint
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/common.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/common.py b/spot-oa/api/graphql/common.py
new file mode 100644
index 0000000..60ac29d
--- /dev/null
+++ b/spot-oa/api/graphql/common.py
@@ -0,0 +1,142 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from collections import OrderedDict
+from datetime import date, datetime
+from graphql import (
+ GraphQLScalarType,
+ GraphQLObjectType,
+ GraphQLField,
+ GraphQLString,
+ GraphQLInt,
+ GraphQLBoolean,
+ GraphQLList,
+ GraphQLNonNull
+)
+import socket
+
+def coerce_date(value):
+ if isinstance(value, date):
+ return value
+ elif isinstance(value, datetime):
+ return value.date()
+ elif isinstance(value, int):
+ return date.utcfromtimestamp(value)
+ else:
+ return datetime.strptime(str(value), '%Y-%m-%d').date()
+
+def serialize_date(value):
+ return datetime.strptime(value, '%Y-%m-%d').strftime('%Y-%m-%d')
+
+def parse_date_literal(ast):
+ return datetime.strptime(ast.value, '%Y-%m-%d')
+
+SpotDateType = GraphQLScalarType(
+ name='SpotDateType',
+ description='The `Date` scalar type represents date values in the format yyyy-mm-dd.',
+ serialize=serialize_date,
+ parse_value=coerce_date,
+ parse_literal=parse_date_literal)
+
+def coerce_datetime(value):
+ if isinstance(value, int):
+ value = datetime.utcfromtimestamp(value)
+ elif not isinstance(value, datetime):
+ value = datetime.strptime(str(value), '%Y-%m-%d %H:%M:%S')
+
+ return value
+
+def serialize_datetime(value):
+ if not isinstance(value, datetime):
+ value = datetime.strptime(str(value), '%Y-%m-%d %H:%M:%S')
+
+ return value.strftime('%Y-%m-%d %H:%M:%S')
+
+def parse_datetime_literal(ast):
+ return datetime.strptime(ast.value, '%Y-%m-%d %H:%M:%S')
+
+SpotDatetimeType = GraphQLScalarType(
+ name='SpotDatetimeType',
+ description='The `Datetime` scalar type represents datetime values in the format yyyy-mm-dd hh:mm:ss.',
+ serialize=serialize_datetime,
+ parse_value=coerce_datetime,
+ parse_literal=parse_datetime_literal)
+
+def coerce_ip(value):
+ return str(value)
+
+def parse_ip_literal(ast):
+ socket.inet_aton(ast.value)
+
+ return ast.value
+
+SpotIpType = GraphQLScalarType(
+ name='SpotIpType',
+ description='The `Ip` scalar type represents a network ip in dot-decimal format.',
+ serialize=coerce_ip,
+ parse_value=coerce_ip,
+ parse_literal=parse_ip_literal)
+
+SpotOperationOutputType = GraphQLObjectType(
+ name='SpotOperationOutputType',
+ fields={
+ 'success': GraphQLField(
+ type=GraphQLNonNull(GraphQLBoolean),
+ description='True after the operation success',
+ resolver=lambda root, *_: root.get('success')
+ )
+ }
+)
+
+def create_spot_node_type(name, extra_fields={}):
+ def get_fields():
+ fields = {
+ 'name': GraphQLField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Node name',
+ resolver=lambda root, *_: root.get('name')
+ ),
+ 'children': GraphQLField(
+ type=GraphQLList(NodeType),
+ description='Children list',
+ resolver=lambda root, *_: root.get('children')
+ )
+ }
+ fields.update(extra_fields if type(extra_fields) is dict else {})
+
+ return fields
+
+ NodeType = GraphQLObjectType(
+ name=name,
+ fields=get_fields
+ )
+
+ return NodeType
+
+IngestSummaryType = GraphQLObjectType(
+ name='SpotIngestSummaryType',
+ description='Number of ingested records',
+ fields={
+ 'datetime': GraphQLField(
+ type=SpotDatetimeType,
+ resolver=lambda root, *_: '{}:00'.format(root.get('tdate'))
+ ),
+ 'total': GraphQLField(
+ type=GraphQLInt,
+ resolver=lambda root, *_: root.get('total')
+ )
+ }
+)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/dns/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/dns/__init__.py b/spot-oa/api/graphql/dns/__init__.py
new file mode 100644
index 0000000..d5c4f98
--- /dev/null
+++ b/spot-oa/api/graphql/dns/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from query import QueryType, TYPES
+from mutation import MutationType
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/dns/mutation.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/dns/mutation.py b/spot-oa/api/graphql/dns/mutation.py
new file mode 100644
index 0000000..e76b596
--- /dev/null
+++ b/spot-oa/api/graphql/dns/mutation.py
@@ -0,0 +1,182 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date
+from graphql import (
+ GraphQLObjectType,
+ GraphQLField,
+ GraphQLArgument,
+ GraphQLString,
+ GraphQLInt,
+ GraphQLNonNull,
+ GraphQLList,
+ GraphQLInputObjectType,
+ GraphQLInputObjectField
+)
+
+from api.graphql.common import SpotDateType, SpotIpType, SpotOperationOutputType
+import api.resources.dns as Dns
+
+ScoreInputType = GraphQLInputObjectType(
+ name='DnsScoreType',
+ fields={
+ 'date': GraphQLInputObjectField(
+ type=SpotDateType,
+ description='A reference date for the scoring process. Defaults to today'
+ ),
+ 'score': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLInt),
+ description='A score value, 1->High, 2->Medium, 3->Low'
+ ),
+ 'dnsQuery': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='Dns query name to score'
+ ),
+ 'clientIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Client IP to score'
+ )
+ }
+)
+
+ThreatDetailsInputType = GraphQLInputObjectType(
+ name='DnsThreatDetailsInputType',
+ fields={
+ 'total': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='The number of time an IP sent a dns query'
+ ),
+ 'dnsQuery': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='DNS query name'
+ ),
+ 'clientIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Client IP address'
+ )
+ }
+)
+
+CreateStoryboardInputType = GraphQLInputObjectType(
+ name='DnsCreateStoryboardInputType',
+ fields={
+ 'date': GraphQLInputObjectField(
+ type=SpotDateType,
+ description='A reference date for the storyboard being created. Defaults to today'
+ ),
+ 'dnsQuery': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='Threat dns query name'
+ ),
+ 'clientIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Threat client IP'
+ ),
+ 'title': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat title'
+ ),
+ 'text': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat title description'
+ ),
+ 'threatDetails': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(ThreatDetailsInputType))),
+ description='Threat details. See DnsThreatInformation.details'
+ )
+ }
+)
+
+def _score_records(args):
+ results = []
+
+ _input = args.get('input')
+ for cmd in _input:
+ _date = cmd.get('date', date.today())
+ dns_query = cmd.get('dnsQuery', '')
+ client_ip = cmd.get('clientIp', '')
+ query_score = cmd.get('score') if dns_query else 0
+ client_ip_score = cmd.get('score') if client_ip else 0
+
+ result = Dns.score_connection(date=_date, dns=dns_query, ip=client_ip, dns_sev=query_score, ip_sev=client_ip_score)
+
+ results.append({'success': result})
+
+ return results
+
+def _create_storyboard(args):
+ _input = args.get('input')
+ _date = _input.get('date', date.today())
+ dns_query = _input.get('dnsQuery', '')
+ client_ip = _input.get('clientIp', '')
+ threat_details = _input.get('threatDetails')
+ title = _input.get('title')
+ text = _input.get('text')
+
+ result = Dns.create_storyboard(
+ date=_date, query=dns_query, ip=client_ip,
+ title=title, text=text,
+ expanded_search=threat_details)
+
+ return {'success': result}
+
+
+def _reset_scored_connections(args):
+ _date = args.get('date', date.today())
+
+ result = Dns.reset_scored_connections(date=_date)
+
+ return {'success': result}
+
+
+MutationType = GraphQLObjectType(
+ name='DnsMutationType',
+ fields={
+ 'score': GraphQLField(
+ type=GraphQLList(SpotOperationOutputType),
+ description='Sets a score value to connections',
+ args={
+ 'input': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(ScoreInputType))),
+ description='Score criteria'
+ )
+ },
+ resolver=lambda root, args, *_: _score_records(args)
+ ),
+ 'createStoryboard': GraphQLField(
+ type=SpotOperationOutputType,
+ description='Request Spot to create an entry on storyboard for a particular threat',
+ args={
+ 'input': GraphQLArgument(
+ type=GraphQLNonNull(CreateStoryboardInputType),
+ description='Threat information'
+ )
+ },
+ resolver=lambda root, args, *_: _create_storyboard(args)
+ ),
+ 'resetScoredConnections': GraphQLField(
+ type=SpotOperationOutputType,
+ description='Resets all scored connections for a certain day',
+ args={
+ 'date': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='Date to clean'
+ )
+ },
+ resolver=lambda root, args, *_: _reset_scored_connections(args)
+ )
+ }
+)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/dns/query.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/dns/query.py b/spot-oa/api/graphql/dns/query.py
new file mode 100644
index 0000000..84f280b
--- /dev/null
+++ b/spot-oa/api/graphql/dns/query.py
@@ -0,0 +1,519 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date, datetime
+from graphql import (
+ GraphQLObjectType,
+ GraphQLField,
+ GraphQLArgument,
+ GraphQLNonNull,
+ GraphQLList,
+ GraphQLString,
+ GraphQLInt,
+ GraphQLFloat,
+ GraphQLUnionType,
+ GraphQLInterfaceType
+)
+
+from api.graphql.common import SpotDateType, SpotDatetimeType, SpotIpType, IngestSummaryType
+import api.resources.dns as Dns
+
+SuspiciousType = GraphQLObjectType(
+ name='DnsSuspiciousType',
+ fields={
+ 'frameTime': GraphQLField(
+ type=SpotDatetimeType,
+ description='Frame time',
+ resolver=lambda root, *_: datetime.utcfromtimestamp(int(root.get('unix_tstamp') or 0))
+ ),
+ 'frameLength': GraphQLField(
+ type=GraphQLInt,
+ description='Frame length in bytes',
+ resolver=lambda root, *_: root.get('frame_len')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('ip_dst')
+ ),
+ 'dnsQuery': GraphQLField(
+ type=GraphQLString,
+ description='DNS query name',
+ resolver=lambda root, *_: root.get('dns_qry_name')
+ ),
+ 'dnsQueryClass': GraphQLField(
+ type=GraphQLInt,
+ description='DNS query class',
+ resolver=lambda root, *_: int(root.get('dns_qry_class') or '0x0', 16)
+ ),
+ 'dnsQueryType': GraphQLField(
+ type=GraphQLInt,
+ description='DNS query type',
+ resolver=lambda root, *_: root.get('dns_qry_type') or 0
+ ),
+ 'dnsQueryRcode': GraphQLField(
+ type=GraphQLInt,
+ description='DNS query response code',
+ resolver=lambda root, *_: root.get('dns_qry_rcode') or 0
+ ),
+ 'score': GraphQLField(
+ type=GraphQLFloat,
+ description='Spot ML score value',
+ resolver=lambda root, *_: root.get('ml_score') or 0
+ ),
+ 'tld': GraphQLField(
+ type=GraphQLString,
+ description='Top Level Domain',
+ resolver=lambda root, *_: root.get('tld')
+ ),
+ 'dnsQueryRep': GraphQLField(
+ type=GraphQLString,
+ description='DNS query name reputation metadata',
+ resolver=lambda root, *_: root.get('query_rep')
+ ),
+ 'clientIpSev': GraphQLField(
+ type=GraphQLInt,
+ description='Client IP user risk score',
+ resolver=lambda root, *_: root.get('ip_sev') or 0
+ ),
+ 'dnsQuerySev': GraphQLField(
+ type=GraphQLInt,
+ description='@deprecated DNS query name user risk score',
+ resolver=lambda root, *_: root.get('dns_sev') or 0
+ ),
+ 'dnsQueryClassLabel': GraphQLField(
+ type=GraphQLString,
+ description='DNS query class name',
+ resolver=lambda root, *_: root.get('dns_qry_class_name')
+ ),
+ 'dnsQueryTypeLabel': GraphQLField(
+ type=GraphQLString,
+ description='DNS query type name',
+ resolver=lambda root, *_: root.get('dns_qry_type_name')
+ ),
+ 'dnsQueryRcodeLabel': GraphQLField(
+ type=GraphQLString,
+ description='DNS query response code name',
+ resolver=lambda root, *_: root.get('dns_qry_rcode_name')
+ ),
+ 'networkContext': GraphQLField(
+ type=GraphQLString,
+ description='@deprecated Network context for client ip',
+ resolver=lambda root, *_: root.get('network_context')
+ ),
+ 'unixTimestamp': GraphQLField(
+ type=GraphQLInt,
+ description='Frame unix timestamp',
+ resolver=lambda root, *_: root.get('unix_tstamp') or 0
+ )
+ }
+)
+
+EdgeDetailsType = GraphQLObjectType(
+ name='DnsEdgeDetailsType',
+ fields={
+ 'frameTime': GraphQLField(
+ type=SpotDatetimeType,
+ description='Frame time',
+ resolver=lambda root, *_: datetime.utcfromtimestamp(int(root.get('unix_tstamp') or 0))
+ ),
+ 'frameLength': GraphQLField(
+ type=GraphQLInt,
+ description='Frame length in bytes',
+ resolver=lambda root, *_: root.get('frame_len')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('ip_dst')
+ ),
+ 'serverIp': GraphQLField(
+ type=SpotIpType,
+ description='DNS server IP address',
+ resolver=lambda root, *_: root.get('ip_src')
+ ),
+ 'dnsQuery': GraphQLField(
+ type=GraphQLString,
+ description='DNS query name',
+ resolver=lambda root, *_: root.get('dns_qry_name')
+ ),
+ 'dnsQueryClass': GraphQLField(
+ type=GraphQLInt,
+ description='DNS query class',
+ resolver=lambda root, *_: int(root.get('dns_qry_class') or '0x0', 16)
+ ),
+ 'dnsQueryType': GraphQLField(
+ type=GraphQLInt,
+ description='DNS query type',
+ resolver=lambda root, *_: root.get('dns_qry_type') or 0
+ ),
+ 'dnsQueryRcode': GraphQLField(
+ type=GraphQLInt,
+ description='DNS query response code',
+ resolver=lambda root, *_: root.get('dns_qry_rcode') or 0
+ ),
+ 'dnsQueryClassLabel': GraphQLField(
+ type=GraphQLString,
+ description='DNS query class name',
+ resolver=lambda root, *_: root.get('dns_qry_class_name')
+ ),
+ 'dnsQueryTypeLabel': GraphQLField(
+ type=GraphQLString,
+ description='DNS query type name',
+ resolver=lambda root, *_: root.get('dns_qry_type_name')
+ ),
+ 'dnsQueryRcodeLabel': GraphQLField(
+ type=GraphQLString,
+ description='DNS query response code name',
+ resolver=lambda root, *_: root.get('dns_qry_rcode_name')
+ ),
+ 'dnsQueryAnswers': GraphQLField(
+ type=GraphQLList(GraphQLString),
+ description='DNS answers',
+ resolver=lambda root, *_: root.get('dns_a', '').split('|')
+ ),
+ 'unixTimestamp': GraphQLField(
+ type=GraphQLInt,
+ description='Frame unix timestamp',
+ resolver=lambda root, *_: root.get('unix_tstamp') or 0
+ )
+ }
+)
+
+ThreatDetailsType = GraphQLObjectType(
+ name='DnsThreatDetailsType',
+ fields={
+ 'total': GraphQLField(
+ type=GraphQLInt,
+ description='The number of time an IP sent a dns query',
+ resolver=lambda root, *_: root.get('total')
+ ),
+ 'dnsQuery': GraphQLField(
+ type=GraphQLString,
+ description='DNS query name',
+ resolver=lambda root, *_: root.get('dns_qry_name')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client IP address',
+ resolver=lambda root, *_: root.get('ip_dst')
+ )
+ }
+)
+
+IpDetailsType = GraphQLObjectType(
+ name='DnsIpDetailsType',
+ fields={
+ 'dnsQuery': GraphQLField(
+ type=GraphQLString,
+ description='DNS query name',
+ resolver=lambda root, *_: root.get('dns_qry_name')
+ ),
+ 'dnsQueryAnswers': GraphQLField(
+ type=GraphQLList(GraphQLString),
+ description='DNS answers',
+ resolver=lambda root, *_: root.get('dns_a', '').split('|')
+ )
+ }
+)
+
+ScoredThreatType = GraphQLObjectType(
+ name='DnsScoredThreatType',
+ fields={
+ 'datetime': GraphQLField(
+ type=SpotDatetimeType,
+ description='Date and time of user score',
+ resolver=lambda root, *_: datetime.utcfromtimestamp(int(root.get('unix_tstamp') or 0))
+ ),
+ 'dnsQuery': GraphQLField(
+ type=GraphQLString,
+ description='DNS query name',
+ resolver=lambda root, *_: root.get('dns_qry_name')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('ip_dst')
+ ),
+ 'dnsScore': GraphQLField(
+ type=GraphQLInt,
+ description='DNS query name risk score value. 1->High, 2->Medium, 3->Low',
+ resolver=lambda root, *_: root.get('dns_sev') or 0
+ ),
+ 'clientIpScore': GraphQLField(
+ type=GraphQLInt,
+ description='Client IP address risk score value. 1->High, 2->Medium, 3->Low',
+ resolver=lambda root, *_: root.get('ip_sev') or 0
+ )
+ }
+)
+
+CommentInterface = GraphQLInterfaceType(
+ name='DnsCommentInterface',
+ fields={
+ 'title': GraphQLField(GraphQLString),
+ 'text': GraphQLField(GraphQLString)
+ },
+ resolve_type=lambda root, *_: QueryCommentType if root.get('dns_threat') else ClientIpCommentType
+)
+
+QueryCommentType = GraphQLObjectType(
+ name='DnsQueryCommentType',
+ interfaces=[CommentInterface],
+ fields={
+ 'dnsQuery': GraphQLField(
+ type=GraphQLString,
+ description='High risk DNS query name',
+ resolver=lambda root, *_: root.get('dns_threat')
+ ),
+ 'title': GraphQLField(
+ type=GraphQLString,
+ description='Threat title',
+ resolver=lambda root, *_: root.get('title')
+ ),
+ 'text': GraphQLField(
+ type=GraphQLString,
+ description='Threat description',
+ resolver=lambda root, *_: root.get('text')
+ )
+ }
+)
+
+ClientIpCommentType = GraphQLObjectType(
+ name='DnsClientIpCommentType',
+ interfaces=[CommentInterface],
+ fields={
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='High risk client IP address',
+ resolver=lambda root, *_: root.get('ip_threat')
+ ),
+ 'title': GraphQLField(
+ type=GraphQLString,
+ description='Threat title',
+ resolver=lambda root, *_: root.get('title')
+ ),
+ 'text': GraphQLField(
+ type=GraphQLString,
+ description='Threat description',
+ resolver=lambda root, *_: root.get('text')
+ )
+ }
+)
+
+ThreatsInformationType = GraphQLObjectType(
+ name='DnsThreats',
+ fields={
+ 'list': GraphQLField(
+ type=GraphQLList(ScoredThreatType),
+ description='List of DNS query names or client IPs that have been scored',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of high risk threats. Defaults to today'
+ )
+ },
+ resolver=lambda root, args, *_: Dns.get_scored_connections(date=args.get('date', date.today()))
+ ),
+ 'comments': GraphQLField(
+ type=GraphQLList(CommentInterface),
+ description='A list of comments about threats',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of high risk comments. Defaults to today'
+ )
+ },
+ resolver=lambda root, args, *_: Dns.comments(date=args.get('date', date.today()))
+ )
+ }
+)
+
+IncidentProgressionInterface = GraphQLInterfaceType(
+ name='DnsIncidentProgressionInterface',
+ fields={
+ 'total': GraphQLField(
+ type=GraphQLInt
+ )
+ },
+ resolve_type=lambda root, *_: IncidentProgressionQueryType if root.get('dns_qry_name') else IncidentProgressionClientIpType
+)
+
+IncidentProgressionClientIpType = GraphQLObjectType(
+ name='DnsIncidentProgressionClientIpType',
+ interfaces=[IncidentProgressionInterface],
+ fields={
+ 'total': GraphQLField(
+ type=GraphQLInt,
+ resolver=lambda root, *_: root.get('total')
+ ),
+ 'clientIp': GraphQLField(
+ type=SpotIpType,
+ description='Client\'s IP address',
+ resolver=lambda root, *_: root.get('ip_dst')
+ )
+ }
+)
+
+IncidentProgressionQueryType = GraphQLObjectType(
+ name='DnsIncidentProgressionQueryType',
+ interfaces=[IncidentProgressionInterface],
+ fields={
+ 'total': GraphQLField(
+ type=GraphQLInt,
+ resolver=lambda root, *_: root.get('total')
+ ),
+ 'dnsQuery': GraphQLField(
+ type=GraphQLString,
+ description='DNS query name',
+ resolver=lambda root, *_: root.get('dns_qry_name')
+ )
+ }
+)
+
+ThreatInformationType = GraphQLObjectType(
+ name='DnsThreatInformation',
+ fields={
+ 'incidentProgression': GraphQLField(
+ type=GraphQLList(IncidentProgressionInterface),
+ description='Details the type of connections that conform the activity related to the threat',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for incident progression information. Defaults to today'
+ ),
+ 'dnsQuery': GraphQLArgument(
+ type=GraphQLString,
+ description='Threat\'s dns query'
+ ),
+ 'clientIp': GraphQLArgument(
+ type=SpotIpType,
+ description='Threat\'s client ip'
+ )
+ },
+ resolver=lambda root, args, *_ : Dns.incident_progression(date=args.get('date', date.today()), query=args.get('dnsQuery'), ip=args.get('clientIp'))
+ ),
+ 'details': GraphQLField(
+ type=GraphQLList(ThreatDetailsType),
+ description='Get details about threats',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as a reference for suspicous queries. Defaults to today'
+ ),
+ 'dnsQuery': GraphQLArgument(
+ type=GraphQLString,
+ description='DNS query name of interest'
+ ),
+ 'clientIp': GraphQLArgument(
+ type=SpotIpType,
+ description='Client IP address of interest'
+ ),
+ 'first': GraphQLArgument(
+ type=GraphQLInt,
+ description='The number of records to return'
+ )
+ },
+ resolver=lambda root, args, *_: Dns.expanded_search(
+ date=args.get('date', date.today()),
+ query=args.get('dnsQuery'),
+ ip=args.get('clientIp'),
+ limit=args.get('first'))
+ )
+ }
+)
+
+QueryType = GraphQLObjectType(
+ name='DnsQueryType',
+ fields={
+ 'suspicious': GraphQLField(
+ type=GraphQLList(SuspiciousType),
+ description='Suspicious DNS query names',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as a reference for suspicious connections. Defaults to today'
+ ),
+ 'clientIp': GraphQLArgument(
+ type=SpotIpType,
+ description='Client IP of interest'
+ ),
+ 'dnsQuery': GraphQLArgument(
+ type=GraphQLString,
+ description='Partial query name of interest'
+ )
+ },
+ resolver=lambda root, args, *_: Dns.suspicious_queries(date=args.get('date', date.today()), ip=args.get('clientIp'), query=args.get('dnsQuery'))
+ ),
+ 'edgeDetails': GraphQLField(
+ type=GraphQLList(EdgeDetailsType),
+ description='DNS queries between client and DNS server around a particular moment in time',
+ args={
+ 'frameTime': GraphQLArgument(
+ type=GraphQLNonNull(SpotDatetimeType),
+ description='Time of interest'
+ ),
+ 'dnsQuery': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLString),
+ description='DNS query name of interest'
+ )
+ },
+ resolver=lambda root, args, *_: Dns.details(frame_time=args.get('frameTime'), query=args.get('dnsQuery'))
+ ),
+ 'ipDetails': GraphQLField(
+ type=GraphQLList(IpDetailsType),
+ description='Query names made by client',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as a reference for suspicous connections. Defaults to today'
+ ),
+ 'clientIp': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Client\'s IP address'
+ )
+ },
+ resolver=lambda root, args, *_: Dns.client_details(date=args.get('date', date.today()), ip=args.get('clientIp'))
+ ),
+ 'threats': GraphQLField(
+ type=ThreatsInformationType,
+ description='Advanced inforamtion about threats',
+ resolver=lambda *_ : {}
+ ),
+ 'threat': GraphQLField(
+ type=ThreatInformationType,
+ description='Advanced inforamtion about a single threat',
+ resolver=lambda *_: {}
+ ),
+ 'ingestSummary': GraphQLField(
+ type=GraphQLList(IngestSummaryType),
+ description='Summary of ingested DNS records in range',
+ args={
+ 'startDate': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='Start date'
+ ),
+ 'endDate': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='End date'
+ )
+ },
+ resolver=lambda root, args, *_: Dns.ingest_summary(start_date=args.get('startDate'), end_date=args.get('endDate'))
+ )
+ }
+)
+
+TYPES = [QueryCommentType, ClientIpCommentType, IncidentProgressionQueryType, IncidentProgressionClientIpType]
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/flow/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/flow/__init__.py b/spot-oa/api/graphql/flow/__init__.py
new file mode 100644
index 0000000..d5c4f98
--- /dev/null
+++ b/spot-oa/api/graphql/flow/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from query import QueryType, TYPES
+from mutation import MutationType
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/flow/mutation.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/flow/mutation.py b/spot-oa/api/graphql/flow/mutation.py
new file mode 100644
index 0000000..32c4aea
--- /dev/null
+++ b/spot-oa/api/graphql/flow/mutation.py
@@ -0,0 +1,218 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date
+from graphql import (
+ GraphQLObjectType,
+ GraphQLField,
+ GraphQLArgument,
+ GraphQLString,
+ GraphQLInt,
+ GraphQLNonNull,
+ GraphQLList,
+ GraphQLInputObjectType,
+ GraphQLInputObjectField
+)
+
+from api.graphql.common import SpotDateType, SpotIpType, SpotOperationOutputType
+from api.graphql.common import SpotDateType, SpotDatetimeType, SpotIpType, SpotOperationOutputType
+import api.resources.flow as Flow
+
+ScoreInputType = GraphQLInputObjectType(
+ name='NetflowScoreInputType',
+ fields={
+ 'date': GraphQLInputObjectField(
+ type=SpotDateType,
+ description='A reference date for the score process. Defaults to today'
+ ),
+ 'score': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLInt),
+ description='A score value, 1->High, 2->Medium, 3->Low'
+ ),
+ 'srcIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Source IP to score'
+ ),
+ 'dstIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Destination IP to score'
+ ),
+ 'srcPort': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Source port to score'
+ ),
+ 'dstPort': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Destination port to score'
+ )
+ }
+)
+
+ThreatDetailsInputType = GraphQLInputObjectType(
+ name='NetflowThreatDetailsInputType',
+ fields={
+ 'firstSeen': GraphQLInputObjectField(
+ type=SpotDatetimeType,
+ description='First time two IPs were seen on a particular day of flow traffic data'
+ ),
+ 'lastSeen': GraphQLInputObjectField(
+ type=SpotDatetimeType,
+ description='Last time two IPs were seen on a particular day of flow traffic data'
+ ),
+ 'srcIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Source IP address'
+ ),
+ 'dstIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Destination IP address'
+ ),
+ 'srcPort': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Source port'
+ ),
+ 'dstPort': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Destination port'
+ ),
+ 'connections': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Number of connections on a particular day of flow traffic data'
+ ),
+ 'maxPkts': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Maximum number of packets tranferred on a single connection'
+ ),
+ 'avgPkts': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Average number of packets transferred bwteen IPs'
+ ),
+ 'maxBytes': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Maximum number of bytes tranferred on a single connection'
+ ),
+ 'avgBytes': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Average number of bytes transferred bwteen IPs'
+ )
+ }
+)
+
+CreateStoryboardInputType = GraphQLInputObjectType(
+ name='NetflowCreateStoryboardInputType',
+ fields={
+ 'date': GraphQLInputObjectField(
+ type=SpotDateType,
+ description='A reference date for the storyboard being created. Defaults to today'
+ ),
+ 'ip': GraphQLInputObjectField(
+ type=GraphQLNonNull(SpotIpType),
+ description='High risk IP address'
+ ),
+ 'title': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat title'
+ ),
+ 'text': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat title description'
+ ),
+ 'threatDetails': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLList(ThreatDetailsInputType)),
+ description='Threat details. See NetflowThreatInformation.details'
+ ),
+ 'first': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='The number of records to return'
+ )
+ }
+)
+
+def _score_connection(args):
+ results = []
+
+ _input = args.get('input')
+ for cmd in _input:
+ result = Flow.score_connection(
+ date=cmd['date'], score=cmd['score'],
+ src_ip=cmd.get('srcIp'), src_port=cmd.get('srcPort'),
+ dst_ip=cmd.get('dstIp'), dst_port=cmd.get('dstPort')
+ )
+
+ results.append({'success': result})
+
+ return results
+
+def _create_storyboard(args):
+ _input = args.get('input')
+ _date = _input.get('date', date.today())
+ ip = _input.get('ip')
+ threat_details = _input.get('threatDetails')
+ title = _input.get('title')
+ text = _input.get('text')
+ first = _input.get('first')
+
+ result = Flow.create_storyboard(date=_date, ip=ip, title=title, text=text, expanded_search=threat_details, top_results=first)
+
+ return {'success': result}
+
+
+def _reset_scored_connections(args):
+ _date = args.get('date', date.today())
+
+ result = Flow.reset_scored_connections(date=_date)
+
+ return {'success': result}
+
+
+MutationType = GraphQLObjectType(
+ name='NetflowMutationType',
+ fields={
+ 'score': GraphQLField(
+ type=GraphQLList(SpotOperationOutputType),
+ description='Sets a score value to connections',
+ args={
+ 'input': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(ScoreInputType))),
+ description='Score criteria'
+ )
+ },
+ resolver=lambda root, args, *_: _score_connection(args)
+ ),
+ 'createStoryboard': GraphQLField(
+ type=SpotOperationOutputType,
+ description='Request Spot to create an entry on storyboard for a particular threat',
+ args={
+ 'input': GraphQLArgument(
+ type=GraphQLNonNull(CreateStoryboardInputType),
+ description='Threat information'
+ )
+ },
+ resolver=lambda root, args, *_: _create_storyboard(args)
+ ),
+ 'resetScoredConnections': GraphQLField(
+ type=SpotOperationOutputType,
+ description='Resets all scored connections for a certain day',
+ args={
+ 'date': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='Date to clean'
+ )
+ },
+ resolver=lambda root, args, *_: _reset_scored_connections(args)
+ )
+ }
+)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/flow/query.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/flow/query.py b/spot-oa/api/graphql/flow/query.py
new file mode 100644
index 0000000..4435176
--- /dev/null
+++ b/spot-oa/api/graphql/flow/query.py
@@ -0,0 +1,691 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date
+from graphql import (
+ GraphQLObjectType,
+ GraphQLField,
+ GraphQLArgument,
+ GraphQLList,
+ GraphQLString,
+ GraphQLInt,
+ GraphQLFloat,
+ GraphQLBoolean,
+ GraphQLNonNull
+)
+
+from api.graphql.common import SpotDateType, SpotDatetimeType, SpotIpType, create_spot_node_type, IngestSummaryType
+import api.resources.flow as Flow
+
+SuspiciousType = GraphQLObjectType(
+ name='NetflowSuspiciousType',
+ fields={
+ 'tstart': GraphQLField(
+ type=SpotDatetimeType,
+ description='Time the flow was received by the flow collector',
+ resolver=lambda root, *_: root.get('tstart')
+ ),
+ 'srcIp': GraphQLField(
+ type=GraphQLString,
+ description='Source IP address',
+ resolver=lambda root, *_: root.get('srcip')
+ ),
+ 'dstIp': GraphQLField(
+ type=GraphQLString,
+ description='Destination IP address',
+ resolver=lambda root, *_: root.get('dstip')
+ ),
+ 'srcPort': GraphQLField(
+ type=GraphQLInt,
+ description='Source port',
+ resolver=lambda root, *_: root.get('sport') or 0
+ ),
+ 'dstPort': GraphQLField(
+ type=GraphQLInt,
+ description='Destination port',
+ resolver=lambda root, *_: root.get('dport') or 0
+ ),
+ 'protocol': GraphQLField(
+ type=GraphQLString,
+ description='IP protocol',
+ resolver=lambda root, *_: root.get('proto')
+ ),
+ 'inPkts': GraphQLField(
+ type=GraphQLInt,
+ description='Input packets',
+ resolver=lambda root, *_: root.get('ipkt') or 0
+ ),
+ 'inBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Input bytes',
+ resolver=lambda root, *_: root.get('ibyt') or 0
+ ),
+ 'outPkts': GraphQLField(
+ type=GraphQLInt,
+ description='Output packets',
+ resolver=lambda root, *_: root.get('opkt') or 0
+ ),
+ 'outBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Output bytes',
+ resolver=lambda root, *_: root.get('obyt') or 0
+ ),
+ 'score': GraphQLField(
+ type=GraphQLFloat,
+ description='Spot ML score',
+ resolver=lambda root, *_: root.get('ml_score') or 0
+ ),
+ 'rank': GraphQLField(
+ type=GraphQLInt,
+ description='Spot ML rank',
+ resolver=lambda root, *_: root.get('rank') or 0
+ ),
+ 'srcIp_isInternal': GraphQLField(
+ type=GraphQLInt,
+ description='Internal source IP address context flag',
+ resolver=lambda root, *_: root.get('srcip_internal')
+ ),
+ 'dstIp_isInternal': GraphQLField(
+ type=GraphQLInt,
+ description='Internal destionation IP address context flag',
+ resolver=lambda root, *_: root.get('dstip_internal')
+ ),
+ 'srcIp_geoloc': GraphQLField(
+ type=GraphQLString,
+ description='Source IP geolocation',
+ resolver=lambda root, *_: root.get('src_geoloc')
+ ),
+ 'dstIp_geoloc': GraphQLField(
+ type=GraphQLString,
+ description='Destination IP geolocation',
+ resolver=lambda root, *_: root.get('dst_geoloc')
+ ),
+ 'srcIp_domain': GraphQLField(
+ type=GraphQLString,
+ description='Source IP domain',
+ resolver=lambda root, *_: root.get('src_domain')
+ ),
+ 'dstIp_domain': GraphQLField(
+ type=GraphQLString,
+ description='Destination IP domain',
+ resolver=lambda root, *_: root.get('dst_domain')
+ ),
+ 'srcIp_rep': GraphQLField(
+ type=GraphQLString,
+ description='Source IP reputation metadata',
+ resolver=lambda root, *_: root.get('src_rep')
+ ),
+ 'dstIp_rep': GraphQLField(
+ type=GraphQLString,
+ description='Destination IP reputation metadata',
+ resolver=lambda root, *_: root.get('dst_rep')
+ )
+ }
+)
+
+EdgeDetailsType = GraphQLObjectType(
+ name='NetflowEdgeDetailsType',
+ fields={
+ 'tstart': GraphQLField(
+ type=SpotDatetimeType,
+ description='Time the flow was received by the flow collector',
+ resolver=lambda root, *_: root.get('tstart')
+ ),
+ 'srcIp': GraphQLField(
+ type=GraphQLString,
+ description='Source IP address',
+ resolver=lambda root, *_: root.get('srcip')
+ ),
+ 'dstIp': GraphQLField(
+ type=GraphQLString,
+ description='Destination IP address',
+ resolver=lambda root, *_: root.get('dstip')
+ ),
+ 'srcPort': GraphQLField(
+ type=GraphQLString,
+ description='Source port',
+ resolver=lambda root, *_: root.get('sport')
+ ),
+ 'dstPort': GraphQLField(
+ type=GraphQLString,
+ description='Destination port',
+ resolver=lambda root, *_: root.get('dport')
+ ),
+ 'protocol': GraphQLField(
+ type=GraphQLString,
+ description='IP protocol',
+ resolver=lambda root, *_: root.get('proto')
+ ),
+ 'flags': GraphQLField(
+ type=GraphQLString,
+ description='TCP flags',
+ resolver=lambda root, *_: root.get('flags')
+ ),
+ 'tos': GraphQLField(
+ type=GraphQLString,
+ description='DSCP value',
+ resolver=lambda root, *_: root.get('tos')
+ ),
+ 'inBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Input bytes',
+ resolver=lambda root, *_: root.get('ibyt') or 0
+ ),
+ 'inPkts': GraphQLField(
+ type=GraphQLInt,
+ description='Input packets',
+ resolver=lambda root, *_: root.get('ipkt') or 0
+ ),
+ 'inIface': GraphQLField(
+ type=GraphQLString,
+ description='SNMP input interface id index',
+ resolver=lambda root, *_: root.get('input')
+ ),
+ 'outIface': GraphQLField(
+ type=GraphQLString,
+ description='SNMP output interface id index',
+ resolver=lambda root, *_: root.get('output')
+ ),
+ 'routerIp': GraphQLField(
+ type=GraphQLString,
+ description='Reporting router IP address',
+ resolver=lambda root, *_: root.get('rip')
+ ),
+ 'outBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Output bytes',
+ resolver=lambda root, *_: root.get('obyt') or 0
+ ),
+ 'outPkts': GraphQLField(
+ type=GraphQLInt,
+ description='Output packets',
+ resolver=lambda root, *_: root.get('opkt') or 0
+ )
+ }
+)
+
+IpConnectionDetailsType = GraphQLObjectType(
+ name='NetflowIpConnectionDetailsType',
+ fields={
+ 'srcIp': GraphQLField(
+ type=GraphQLString,
+ description='Source IP address',
+ resolver=lambda root, *_: root.get('srcip')
+ ),
+ 'dstIp': GraphQLField(
+ type=GraphQLString,
+ description='Destination IP address',
+ resolver=lambda root, *_: root.get('dstip')
+ ),
+ 'inBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Input bytes',
+ resolver=lambda root, *_: root.get('ibyt') or 0
+ ),
+ 'inPkts': GraphQLField(
+ type=GraphQLInt,
+ description='Input packets',
+ resolver=lambda root, *_: root.get('ipkt') or 0
+ )
+ }
+)
+
+ScoredConnectionType = GraphQLObjectType(
+ name='NetflowScoredConnectionType',
+ fields={
+ 'tstart': GraphQLField(
+ type=SpotDatetimeType,
+ description='Time the flow was received by the flow collector',
+ resolver=lambda root, *_: root.get('tstart')
+ ),
+ 'srcIp': GraphQLField(
+ type=SpotIpType,
+ description='Source IP address',
+ resolver=lambda root, *_: root.get('srcip')
+ ),
+ 'srcPort': GraphQLField(
+ type=GraphQLInt,
+ description='Source port',
+ resolver=lambda root, *_: root.get('srcport') or 0
+ ),
+ 'dstIp': GraphQLField(
+ type=SpotIpType,
+ description='Destination IP address',
+ resolver=lambda root, *_: root.get('dstip')
+ ),
+ 'dstPort': GraphQLField(
+ type=GraphQLInt,
+ description='Destionation port',
+ resolver=lambda root, *_: root.get('dstport') or 0
+ ),
+ 'score': GraphQLField(
+ type=GraphQLInt,
+ description='Risk score value. 1->High, 2->Medium, 3->Low',
+ resolver=lambda root, *_: root.get('score') or 0
+ )
+ }
+)
+
+ThreatDetailsType = GraphQLObjectType(
+ name='NetflowThreatDetailsType',
+ fields={
+ 'firstSeen': GraphQLField(
+ type=SpotDatetimeType,
+ description='First time two IPs were seen on a particular day of flow traffic data',
+ resolver=lambda root, *_: root.get('firstseen')
+ ),
+ 'lastSeen': GraphQLField(
+ type=SpotDatetimeType,
+ description='Last time two IPs were seen on a particular day of flow traffic data',
+ resolver=lambda root, *_: root.get('lastseen')
+ ),
+ 'srcIp': GraphQLField(
+ type=SpotIpType,
+ description='Source IP address',
+ resolver=lambda root, *_: root.get('srcip')
+ ),
+ 'dstIp': GraphQLField(
+ type=SpotIpType,
+ description='Destination IP address',
+ resolver=lambda root, *_: root.get('dstip')
+ ),
+ 'srcPort': GraphQLField(
+ type=GraphQLInt,
+ description='Source port',
+ resolver=lambda root, *_: root.get('sport')
+ ),
+ 'dstPort': GraphQLField(
+ type=GraphQLInt,
+ description='Destination port',
+ resolver=lambda root, *_: root.get('dport')
+ ),
+ 'connections': GraphQLField(
+ type=GraphQLInt,
+ description='Number of connections on a particular day of flow traffic data',
+ resolver=lambda root, *_: root.get('conns')
+ ),
+ 'maxPkts': GraphQLField(
+ type=GraphQLInt,
+ description='Maximum number of packets tranferred on a single connection',
+ resolver=lambda root, *_: root.get('maxpkts')
+ ),
+ 'avgPkts': GraphQLField(
+ type=GraphQLInt,
+ description='Average number of packets transferred bwteen IPs',
+ resolver=lambda root, *_: root.get('avgpkts')
+ ),
+ 'maxBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Maximum number of bytes tranferred on a single connection',
+ resolver=lambda root, *_: root.get('maxbyts')
+ ),
+ 'avgBytes': GraphQLField(
+ type=GraphQLInt,
+ description='Average number of bytes transferred bwteen IPs',
+ resolver=lambda root, *_: root.get('avgbyts')
+ )
+ }
+)
+
+CommentType = GraphQLObjectType(
+ name='NetflowCommentType',
+ fields={
+ 'ip': GraphQLField(
+ type=SpotIpType,
+ description='High risk IP address',
+ resolver=lambda root, *_: root.get('ip_threat')
+ ),
+ 'title': GraphQLField(
+ type=GraphQLString,
+ description='Threat title',
+ resolver=lambda root, *_: root.get('title')
+ ),
+ 'text': GraphQLField(
+ type=GraphQLString,
+ description='Threat description',
+ resolver=lambda root, *_: root.get('text')
+ )
+ }
+)
+
+ThreatsInformationType = GraphQLObjectType(
+ name='NetflowThreatsType',
+ fields={
+ 'list': GraphQLField(
+ type=GraphQLList(ScoredConnectionType),
+ description='List of suspicious IPs that have been scored',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of scored IPs. Defaults to today'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.get_scored_connections(date=args.get('date', date.today()))
+ ),
+ 'comments': GraphQLField(
+ type=GraphQLList(CommentType),
+ description='A list of comments about threats',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference to retrieve the list of high risk comments. Defaults to today'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.story_board(date=args.get('date', date.today()))
+ )
+ }
+)
+
+IncidentProgressionNodeType = create_spot_node_type(
+ 'NetflowIncidentProgressionNodeType')
+
+ImpactAnalysisNodeType = create_spot_node_type('NetflowImpactAnalysisNodeType', {
+ 'size': GraphQLField(
+ type=GraphQLInt,
+ description='Number of inbound, outbound and two-way connections',
+ resolver=lambda root, *_: root.get('size') or 0
+ )
+})
+
+MapViewGeometryType = GraphQLObjectType(
+ name='NetflowMapViewGeometryType',
+ fields={
+ 'coordinates': GraphQLField(
+ type=GraphQLList(GraphQLFloat),
+ description='Geo latitude and longitude',
+ resolver=lambda root, *_: root.get('coordinates')
+ )
+ }
+)
+
+MapViewPropertiesType = GraphQLObjectType(
+ name='NetflowMapViewPropertiesType',
+ fields={
+ 'ip': GraphQLField(
+ type=SpotIpType,
+ description='IP',
+ resolver=lambda root, *_: root.get('ip')
+ ),
+ 'location': GraphQLField(
+ type=GraphQLString,
+ description='Name of the IP\'s location',
+ resolver=lambda root, *_: root.get('location')
+ ),
+ 'type': GraphQLField(
+ type=GraphQLInt,
+ description='Property type',
+ resolver=lambda root, *_: root.get('type')
+ )
+ }
+)
+
+MapViewIpType = GraphQLObjectType(
+ name='NetflowMapViewIpType',
+ fields={
+ 'geometry': GraphQLField(
+ type=MapViewGeometryType,
+ description='Geolocalization information',
+ resolver=lambda root, *_: root.get('geometry')
+ ),
+ 'properties': GraphQLField(
+ type=MapViewPropertiesType,
+ description='Metadata',
+ resolver=lambda root, *_: root.get('properties')
+ )
+ }
+)
+
+MapViewType = GraphQLObjectType(
+ name='NetflowMapViewType',
+ fields={
+ 'srcIps': GraphQLField(
+ type=GraphQLList(MapViewIpType),
+ description='A list of source IPs',
+ resolver=lambda root, *_: root.get('sourceips', [])
+ ),
+ 'dstIps': GraphQLField(
+ type=GraphQLList(MapViewIpType),
+ description='A list of destination IPs',
+ resolver=lambda root, *_: root.get('destips', [])
+ )
+ }
+)
+
+TimelineType = GraphQLObjectType(
+ name='NetflowTimelineType',
+ fields={
+ 'tstart': GraphQLField(
+ type=GraphQLNonNull(SpotDatetimeType),
+ description='Connection\'s start time',
+ resolver=lambda root, *_: root.get('tstart')
+ ),
+ 'tend': GraphQLField(
+ type=GraphQLNonNull(SpotDatetimeType),
+ description='Connection\'s end time',
+ resolver=lambda root, *_: root.get('tend')
+ ),
+ 'srcIp': GraphQLField(
+ type=GraphQLNonNull(SpotIpType),
+ description='Source IP address',
+ resolver=lambda root, *_: root.get('srcip')
+ ),
+ 'dstIp': GraphQLField(
+ type=GraphQLNonNull(SpotIpType),
+ description='Destination IP address',
+ resolver=lambda root, *_: root.get('dstip')
+ ),
+ 'protocol': GraphQLField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Connection\'s protocol',
+ resolver=lambda root, *_: root.get('proto')
+ ),
+ 'srcPort': GraphQLField(
+ type=GraphQLNonNull(GraphQLInt),
+ description='Source port',
+ resolver=lambda root, *_: root.get('sport')
+ ),
+ 'dstPort': GraphQLField(
+ type=GraphQLNonNull(GraphQLInt),
+ description='Destination port',
+ resolver=lambda root, *_: root.get('dport')
+ ),
+ 'pkts': GraphQLField(
+ type=GraphQLNonNull(GraphQLInt),
+ description='Packets tranferred between IPs',
+ resolver=lambda root, *_: root.get('ipkt')
+ ),
+ 'bytes': GraphQLField(
+ type=GraphQLNonNull(GraphQLInt),
+ description='Bytes tranferred between IPs',
+ resolver=lambda root, *_: root.get('ibyt')
+ )
+ }
+)
+
+ThreatInformationType = GraphQLObjectType(
+ name='NetflowThreatInformation',
+ fields={
+ 'details': GraphQLField(
+ type=GraphQLList(ThreatDetailsType),
+ description='Detailed information about a high risk IP',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for high rist IP information. Defaults to today'
+ ),
+ 'ip': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Suspicious IP'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.expanded_search(date=args.get('date', date.today()), ip=args.get('ip'))
+ ),
+ 'incidentProgression': GraphQLField(
+ type=IncidentProgressionNodeType,
+ description='Details for the type of connections that conform the activity related to the threat',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for incident progression information. Defaults to today'
+ ),
+ 'ip': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Suspicious IP'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.incident_progression(date=args.get('date', date.today()), ip=args.get('ip'))
+ ),
+ 'impactAnalysis': GraphQLField(
+ type=ImpactAnalysisNodeType,
+ description='Contains the number of inbound, outbound and two-way connections found related to the suspicious IP',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for impact analysis information. Defaults to today'
+ ),
+ 'ip': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Suspicious IP'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.impact_analysis(date=args.get('date', date.today()), ip=args.get('ip'))
+ ),
+ 'geoLocalization': GraphQLField(
+ type=MapViewType,
+ description='Gelocalization info about the IPs related to this threat',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for geo localization information. Defaults to today'
+ ),
+ 'ip': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Suspicious IP'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.sc_geo(date=args.get('date', date.today()), ip=args.get('ip'))
+ ),
+ 'timeline': GraphQLField(
+ type=GraphQLList(TimelineType),
+ description='Lists \'clusters\' of inbound connections to the IP, grouped by time; showing an overall idea of the times during the day with the most activity',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for time line information. Defaults to today'
+ ),
+ 'ip': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Suspicious Ip'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.time_line(date=args.get('date', date.today()), ip=args.get('ip'))
+ )
+ }
+)
+
+QueryType = GraphQLObjectType(
+ name='NetflowQueryType',
+ fields={
+ 'suspicious': GraphQLField(
+ type=GraphQLList(SuspiciousType),
+ description='Flow suspicious connections',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as a reference for suspicous connections. Defaults to today'
+ ),
+ 'ip': GraphQLArgument(
+ type=SpotIpType,
+ description='IP of interest'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.suspicious_connections(date=args.get('date', date.today()), ip=args.get('ip'))
+ ),
+ 'edgeDetails': GraphQLField(
+ type=GraphQLList(EdgeDetailsType),
+ description='Flow activity between two IPs around a particular moment in time',
+ args={
+ 'tstart': GraphQLArgument(
+ type=GraphQLNonNull(SpotDatetimeType),
+ description='Time of interest'
+ ),
+ 'srcIp': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Source IP address'
+ ),
+ 'dstIp': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='Destination IP address'
+ )
+ },
+ resolver=lambda root, args, *_: Flow.details(
+ date=args.get('tstart'),
+ src_ip=args.get('srcIp'),
+ dst_ip=args.get('dstIp'))
+ ),
+ 'ipDetails': GraphQLField(
+ type=GraphQLList(IpConnectionDetailsType),
+ description='Flow activity details in between IP of interest and other suspicious IPs',
+ args={
+ 'date': GraphQLArgument(
+ type=SpotDateType,
+ description='A date to use as reference for IP network activity details. Defaults to today'
+ ),
+ 'ip': GraphQLArgument(
+ type=GraphQLNonNull(SpotIpType),
+ description='IP address of interest'
+ )
+ },
+ resolver=lambda root, args, *
+ _: Flow.chord_details(date=args.get('date', date.today()), ip=args.get('ip'))
+ ),
+ 'threats': GraphQLField(
+ type=ThreatsInformationType,
+ description='Advanced information about threats',
+ resolver=lambda *_: {}
+ ),
+ 'threat': GraphQLField(
+ type=ThreatInformationType,
+ description='Advanced information about a single threat',
+ resolver=lambda *_: {}
+ ),
+ 'ingestSummary': GraphQLField(
+ type=GraphQLList(IngestSummaryType),
+ description='Summary of ingested flows in range',
+ args={
+ 'startDate': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='Start date'
+ ),
+ 'endDate': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='End date'
+ )
+ },
+ resolver=lambda root, args, *_: Flow.ingest_summary(start_date=args.get('startDate'), end_date=args.get('endDate'))
+ )
+ }
+)
+
+TYPES = []
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/proxy/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/proxy/__init__.py b/spot-oa/api/graphql/proxy/__init__.py
new file mode 100644
index 0000000..d5c4f98
--- /dev/null
+++ b/spot-oa/api/graphql/proxy/__init__.py
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from query import QueryType, TYPES
+from mutation import MutationType
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/proxy/mutation.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/proxy/mutation.py b/spot-oa/api/graphql/proxy/mutation.py
new file mode 100644
index 0000000..2c32a82
--- /dev/null
+++ b/spot-oa/api/graphql/proxy/mutation.py
@@ -0,0 +1,221 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date
+from graphql import (
+ GraphQLObjectType,
+ GraphQLField,
+ GraphQLArgument,
+ GraphQLList,
+ GraphQLString,
+ GraphQLInt,
+ GraphQLNonNull,
+ GraphQLInputObjectType,
+ GraphQLInputObjectField
+)
+
+from api.graphql.common import SpotDateType, SpotDatetimeType, SpotIpType, SpotOperationOutputType
+import api.resources.proxy as Proxy
+
+ScoreInputType = GraphQLInputObjectType(
+ name='ProxyScoreInputType',
+ fields={
+ 'date': GraphQLInputObjectField(
+ type=SpotDateType,
+ description='A reference date for the score process. Defaults to today'
+ ),
+ 'score': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLInt),
+ description='A score value, 1->High, 2->Medium, 3->Low'
+ ),
+ 'uri': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Full URI'
+ )
+ }
+)
+
+ThreatDetailsInputType = GraphQLInputObjectType(
+ name='ProxyThreatDetailsInputType',
+ fields={
+ 'datetime': GraphQLInputObjectField(
+ type=SpotDatetimeType,
+ description='Start time of the request'
+ ),
+ 'clientIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='Client\'s IP address'
+ ),
+ 'username': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='Username used for authetication'
+ ),
+ 'duration': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Connection duration'
+ ),
+ 'uri': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='The original URI requested'
+ ),
+ 'webCategory': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='Web content categories'
+ ),
+ 'responseCode': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='HTTP response code'
+ ),
+ 'requestMethod': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='HTTP request method'
+ ),
+ 'userAgent': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='Client\'s user agent'
+ ),
+ 'responseContentType': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='HTTP response content type (MIME)'
+ ),
+ 'referer': GraphQLInputObjectField(
+ type=GraphQLString,
+ description='The address of the webpage that linked to the resource being requested'
+ ),
+ 'uriPort': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='URI port'
+ ),
+ 'serverIp': GraphQLInputObjectField(
+ type=SpotIpType,
+ description='The address of the webpage that linked to the resource being requested'
+ ),
+ 'serverToClientBytes': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Number of bytes sent from appliance to client'
+ ),
+ 'clientToServerBytes': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='Number of bytes sent from client to appliance'
+ )
+ }
+)
+
+CreateStoryboardInputType = GraphQLInputObjectType(
+ name='ProxyCreateStoryboardInputType',
+ fields={
+ 'date': GraphQLInputObjectField(
+ type=SpotDateType,
+ description='A reference date for the storyboard being created. Defaults to today'
+ ),
+ 'uri': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat UI'
+ ),
+ 'title': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat title'
+ ),
+ 'text': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLString),
+ description='Threat description'
+ ),
+ 'threatDetails': GraphQLInputObjectField(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(ThreatDetailsInputType))),
+ description='Threat details. See ProxyThreatInformation.details'
+ ),
+ 'first': GraphQLInputObjectField(
+ type=GraphQLInt,
+ description='The number of records to return'
+ )
+ }
+)
+
+def _score_connections(args):
+ results = []
+
+ _input = args.get('input')
+ for cmd in _input:
+ _date = cmd.get('date', date.today())
+ score = cmd.get('score')
+ uri = cmd.get('uri')
+
+ result = Proxy.score_request(date=_date, score=score, uri=uri)
+
+ results.append({'success': result})
+
+ return results
+
+
+def _create_storyboard(args):
+ _input = args.get('input')
+ _date = _input.get('date', date.today())
+ uri = _input.get('uri')
+ title = _input.get('title')
+ text = _input.get('text')
+ threat_details = _input.get('threatDetails')
+ first = _input.get('first')
+
+ result = Proxy.create_storyboard(date=_date, uri=uri, title=title, text=text, expanded_search=threat_details, top_results=first)
+
+ return {'success': result}
+
+
+def _reset_scored_connections(args):
+ _date = args.get('date', date.today())
+
+ result = Proxy.reset_scored_connections(date=_date)
+
+ return {'success': result}
+
+
+MutationType = GraphQLObjectType(
+ name='ProxyMutationType',
+ fields={
+ 'score': GraphQLField(
+ type=GraphQLList(SpotOperationOutputType),
+ description='Sets a score value to connections',
+ args={
+ 'input': GraphQLArgument(
+ type=GraphQLNonNull(GraphQLList(GraphQLNonNull(ScoreInputType))),
+ description='Score criteria'
+ )
+ },
+ resolver=lambda root, args, *_: _score_connections(args)
+ ),
+ 'createStoryboard': GraphQLField(
+ type=SpotOperationOutputType,
+ args={
+ 'input': GraphQLArgument(
+ type=GraphQLNonNull(CreateStoryboardInputType),
+ description='Request Spot to create an entry on storyboard for a particular threat'
+ )
+ },
+ resolver=lambda root, args, *_: _create_storyboard(args)
+ ),
+ 'resetScoredConnections': GraphQLField(
+ type=SpotOperationOutputType,
+ description='Resets all scored connections for a certain day',
+ args={
+ 'date': GraphQLArgument(
+ type=GraphQLNonNull(SpotDateType),
+ description='Date to clean'
+ )
+ },
+ resolver=lambda root, args, *_: _reset_scored_connections(args)
+ )
+ }
+)
[11/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/proxy.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/proxy.py b/spot-oa/api/resources/proxy.py
new file mode 100644
index 0000000..c162bbf
--- /dev/null
+++ b/spot-oa/api/resources/proxy.py
@@ -0,0 +1,467 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import md5
+import api.resources.impala_engine as ImpalaEngine
+import api.resources.hdfs_client as HDFSClient
+from hdfs.util import HdfsError
+import api.resources.configurator as Configuration
+from collections import defaultdict
+import json
+import os
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connectios related to a request name in one hour
+--------------------------------------------------------------------------
+"""
+def suspicious_requests(date,uri=None,ip=None,limit=250):
+
+ db = Configuration.db()
+ proxy_query = ("""
+ SELECT STRAIGHT_JOIN
+ ps.tdate,ps.time,ps.clientip,ps.host,ps.reqmethod,ps.useragent,
+ ps.resconttype,ps.duration,ps.username,ps.webcat,ps.referer,
+ ps.respcode,ps.uriport,ps.uripath,ps.uriquery,ps.serverip,ps.scbytes,
+ ps.csbytes,ps.fulluri,ps.ml_score,ps.uri_rep,ps.respcode_name,
+ ps.network_context
+ FROM
+ {0}.proxy_scores ps
+ LEFT JOIN
+ {0}.proxy_threat_investigation pt
+ ON (ps.fulluri = pt.fulluri)
+ WHERE
+ ps.y={1} AND ps.m={2} AND ps.d={3}
+ AND (pt.fulluri is NULL)
+ """).format(db,date.year,date.month,date.day)
+
+
+ p_filter = ""
+ p_filter += " AND ps.fulluri LIKE '%{0}%'".format(uri) if uri else ""
+ p_filter += " AND ps.clientip = '{0}'".format(ip) if ip else ""
+ p_filter += " ORDER BY ps.ml_score limit {0}".format(limit)
+ proxy_query = proxy_query + p_filter
+ return ImpalaEngine.execute_query_as_list(proxy_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connectios details for one request.
+--------------------------------------------------------------------------
+"""
+def details(date,uri,ip):
+
+ if not uri and not ip:
+ return None
+
+ db = Configuration.db()
+ p_details = ("""
+ SELECT
+ tdate,time,clientIp,host,webcat,respcode,respcode_name
+ ,reqmethod,useragent,resconttype,referer,uriport,serverip
+ ,scbytes,csbytes,fulluri,hh
+ FROM
+ {0}.proxy_edge
+ WHERE
+ y={1} AND m={2} AND d={3} AND
+ (fulluri='{4}' AND clientIp='{5}')
+ """).format(db,date.year,date.month,date.day,uri.replace("'","//'"),ip)
+ return ImpalaEngine.execute_query_as_list(p_details)
+
+"""
+--------------------------------------------------------------------------
+Score a request
+--------------------------------------------------------------------------
+"""
+def score_request(date,score,uri):
+
+ if not score and not uri:
+ return None
+
+ db = Configuration.db()
+ p_query = ("""
+ SELECT
+ tdate,time,clientip,host,reqmethod,useragent,resconttype
+ ,duration,username,webcat,referer,respcode,uriport
+ ,uripath,uriquery,serverip,scbytes,csbytes,fulluri
+ ,word,ml_score,uri_rep,respcode_name,network_context
+ FROM
+ {0}.proxy_scores
+ WHERE
+ y={1} and m={2} and d={3}
+ AND fulluri = '{4}'
+ """).format(db,date.year,date.month,date.day,uri)
+
+ connections = ImpalaEngine.execute_query(p_query)
+
+ # add score to connections
+ insert_command = ("""
+ INSERT INTO {0}.proxy_threat_investigation PARTITION (y={1},m={2},d={3})
+ VALUES (""") \
+ .format(db,date.year,date.month,date.day)
+
+ fb_data = []
+ first = True
+ num_rows = 0
+ for row in connections:
+ cip_index = row[2]
+ uri_index = row[18]
+ tme_index = row[2]
+ hash_field = [str( md5.new(str(cip_index) + str(uri_index)).hexdigest() \
+ + str((tme_index.split(":"))[0]) )]
+
+ threat_data = (row[0],row[18],score)
+ fb_data.append([row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7] \
+ ,row[8],row[9],row[10],row[11],row[12],row[13],row[14],row[15] \
+ ,row[16],row[17],row[18],row[19],score,row[20],row[21],row[22], \
+ row[23],hash_field])
+ insert_command += "{0}{1}".format("," if not first else "", threat_data)
+ first = False
+ num_rows += 1
+
+ insert_command += ")"
+ if num_rows > 0: ImpalaEngine.execute_query(insert_command)
+
+ # create feedback file.
+ app_path = Configuration.spot()
+ feedback_path = "{0}/proxy/scored_results/{1}{2}{3}/feedback"\
+ .format(app_path,date.year,str(date.month).zfill(2),str(date.day).zfill(2))
+
+ ap_file = True
+ if len(HDFSClient.list_dir(feedback_path)) == 0:
+ fb_data.insert(0,["p_date","p_time","clientip","host","reqmethod",\
+ "useragent","resconttype","duration","username","webcat","referer",\
+ "respcode","uriport","uripath","uriquery","serverip","scbytes","csbytes",\
+ "fulluri","word","score","uri_rep","uri_sev","respcode_name",\
+ "network_context","hash"])
+ ap_file = False
+
+ HDFSClient.put_file_csv(fb_data,feedback_path,"ml_feedback.csv",append_file=ap_file)
+ return True
+
+"""
+--------------------------------------------------------------------------
+Get expanded search from raw table.
+--------------------------------------------------------------------------
+"""
+def expanded_search(date,uri):
+
+ db = Configuration.db()
+ expanded_query = ("""
+ SELECT p_date, p_time, clientip, username, duration, fulluri,\
+ webcat, respcode, reqmethod,useragent, resconttype,\
+ referer, uriport, serverip, scbytes, csbytes
+ FROM {0}.proxy
+ WHERE y='{1}' AND m='{2}' AND d='{3}'
+ AND (fulluri='{4}' OR referer ='{4}')
+ ORDER BY p_time
+ """)\
+ .format(db,date.year,str(date.month).zfill(2),str(date.day).zfill(2),uri)
+ return ImpalaEngine.execute_query_as_list(expanded_query)
+
+"""
+--------------------------------------------------------------------------
+Get scored request from threat investigation.
+--------------------------------------------------------------------------
+"""
+def get_scored_requests(date):
+
+ db = Configuration.db()
+ sc_query = ("""
+ SELECT
+ tdate,fulluri,uri_sev
+ FROM
+ {0}.proxy_threat_investigation
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ return ImpalaEngine.execute_query_as_list(sc_query)
+
+"""
+--------------------------------------------------------------------------
+Create storyboard.
+Migrated from IPython Notebooks
+--------------------------------------------------------------------------
+"""
+def create_storyboard(uri,date,title,text,expanded_search,top_results):
+
+ clientips = defaultdict(int)
+ reqmethods = defaultdict(int)
+ rescontype = defaultdict(int)
+ referers = defaultdict(int)
+ refered = defaultdict(int)
+ requests = []
+
+
+ for row in expanded_search:
+ clientips[row['clientIp']]+=1
+ reqmethods[row['requestMethod']]+=1
+ rescontype[row['responseContentType']]+=1
+ if row['uri'] == uri:
+ #Source URI's that refered the user to the threat
+ referers[row['referer']]+=1
+ requests += [{'clientip':row['clientIp'], 'referer':row['referer'],'reqmethod':row['requestMethod'], 'resconttype':row['responseContentType']}]
+
+ else:
+ #Destination URI's refered by the threat
+ refered[row['uri']]+=1
+
+ create_incident_progression(uri,requests,refered,date)
+ create_timeline(uri,clientips,date,top_results)
+ save_comments(uri,title,text,date)
+
+ return True
+
+"""
+--------------------------------------------------------------------------
+Create timeline for storyboard
+--------------------------------------------------------------------------
+"""
+def create_timeline(anchor,clientips,date,top_results):
+ response = ""
+ susp_ips = []
+
+ if clientips:
+ srtlist = sorted(list(clientips.items()), key=lambda x: x[1], reverse=True)
+ for val in srtlist[:top_results]:
+ susp_ips.append(val[0])
+
+ if anchor != "":
+ db = Configuration.db()
+ time_line_query = ("""
+ SELECT p_threat,tstart,tend,duration,clientip,respcode,respcodename
+ FROM {0}.proxy_timeline
+ WHERE
+ y={1} AND m={2} AND d={3} AND p_threat != '{4}'
+ """).format(db,date.year,date.month,date.day,anchor.replace("'","//'"))
+
+ tmp_timeline_data = ImpalaEngine.execute_query_as_list(time_line_query)
+
+ imp_query = ("""
+ INSERT INTO TABLE {0}.proxy_timeline
+ PARTITION (y={2}, m={3},d={4})
+ SELECT
+ '{7}' as p_threat, concat(cast(p_date as string),
+ ' ', cast(MIN(p_time) as string)) AS tstart,
+ concat(cast(p_date as string), ' ',
+ cast(MAX(p_time) as string)) AS tend,
+ SUM(duration) AS duration,
+ clientip, respcode,"respCodeName" as respCodeName
+ FROM {0}.proxy
+ WHERE fulluri='{1}' AND clientip IN ({5})
+ AND y='{2}' AND m='{3}' AND d='{4}'
+ GROUP BY clientip, p_time, respcode, p_date
+ LIMIT {6}
+ """)\
+ .format(db,anchor,date.year,str(date.month).zfill(2),\
+ str(date.day).zfill(2),("'" + "','".join(susp_ips) + "'")\
+ ,top_results,anchor)
+
+ app_path = Configuration.spot()
+ old_file = "{0}/proxy/hive/oa/timeline/y={1}/m={2}/d={3}"\
+ .format(app_path,date.year,date.month,date.day)
+
+ HDFSClient.delete_folder(old_file,"impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+
+ #Insert temporary values
+ for item in tmp_timeline_data:
+ insert_query = ("""
+ INSERT INTO {0}.proxy_timeline PARTITION(y={1} , m={2} ,d={3})
+ VALUES ('{4}', '{5}', '{6}',{7},'{8}','{9}','{10}')
+ """)\
+ .format(db,date.year,date.month,date.day,\
+ item["p_threat"],item["tstart"],item["tend"],item["duration"],item["clientip"],item["respcode"],item["respcodename"])
+
+ ImpalaEngine.execute_query(insert_query)
+
+ ImpalaEngine.execute_query(imp_query)
+ response = "Timeline successfully saved"
+ else:
+ response = "Timeline couldn't be created"
+
+"""
+--------------------------------------------------------------------------
+Create inciden progression for storyboard.
+--------------------------------------------------------------------------
+"""
+def create_incident_progression(anchor,requests,referers,date):
+
+ hash_name = md5.new(str(anchor)).hexdigest()
+ file_name = "incident-progression-{0}.json".format(hash_name)
+ app_path = Configuration.spot()
+ hdfs_path = "{0}/proxy/oa/storyboard/{1}/{2}/{3}"\
+ .format(app_path,date.year,date.month,date.day)
+
+ data = {'fulluri':anchor, 'requests':requests,'referer_for':referers.keys()}
+ if HDFSClient.put_file_json(data,hdfs_path,file_name,overwrite_file=True) :
+ response = "Incident progression successfuly created"
+ else:
+ return False
+
+"""
+--------------------------------------------------------------------------
+Save comments for storyboard.
+--------------------------------------------------------------------------
+"""
+def save_comments(uri,title,text,date):
+
+ db = Configuration.db()
+ sb_query = ("""
+ SELECT
+ p_threat,title,text
+ FROM
+ {0}.proxy_storyboard
+ WHERE
+ y = {1} AND m= {2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+ sb_data = ImpalaEngine.execute_query_as_list(sb_query)
+
+
+ # find value if already exists.
+ saved = False
+ for item in sb_data:
+ if item["p_threat"] == uri:
+ item["title"] = title
+ item["text"] = text
+ saved = True
+
+ if not saved:
+ sb_data.append({'text': text, 'p_threat': str(uri), 'title': title})
+
+ #remove old file.
+ app_path = Configuration.spot()
+ old_file = "{0}/proxy/hive/oa/storyboard/y={1}/m={2}/d={3}/"\
+ .format(app_path,date.year,date.month,date.day)
+ HDFSClient.delete_folder(old_file,"impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+
+ for item in sb_data:
+ insert_query = ("""
+ INSERT INTO {0}.proxy_storyboard PARTITION(y={1} , m={2} ,d={3})
+ VALUES ( '{4}', '{5}', '{6}')
+ """)\
+ .format(db,date.year,date.month,date.day,\
+ item["p_threat"],item["title"],item["text"])
+
+ ImpalaEngine.execute_query(insert_query)
+
+"""
+--------------------------------------------------------------------------
+Get storyboard comments.
+--------------------------------------------------------------------------
+"""
+def story_board(date):
+
+ db = Configuration.db()
+ sb_query= ("""
+ SELECT
+ p_threat,title,text
+ FROM
+ {0}.proxy_storyboard
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ results = ImpalaEngine.execute_query_as_list(sb_query)
+ for row in results:
+ row["text"] = row["text"].replace("\n","\\n")
+ return results
+
+"""
+--------------------------------------------------------------------------
+Get timeline for storyboard.
+--------------------------------------------------------------------------
+"""
+def time_line(date,uri):
+
+ db = Configuration.db()
+ time_line_query = ("""
+ SELECT
+ p_threat,tstart,tend,duration,clientip,respcode,respcodename
+ FROM {0}.proxy_timeline
+ WHERE
+ y={1} AND m={2} AND d={3}
+ AND p_threat = '{4}'
+ """).format(db,date.year,date.month,date.day,uri)
+
+ return ImpalaEngine.execute_query_as_list(time_line_query)
+
+"""
+--------------------------------------------------------------------------
+Get incident progression for storyboard.
+--------------------------------------------------------------------------
+"""
+def incident_progression(date,uri):
+
+ app_path = Configuration.spot()
+ hdfs_path = "{0}/proxy/oa/storyboard/{1}/{2}/{3}".format(app_path,\
+ date.year,date.month,date.day)
+
+ hash_name = md5.new(str(uri)).hexdigest()
+ file_name = "incident-progression-{0}.json".format(hash_name)
+
+ if HDFSClient.file_exists(hdfs_path,file_name):
+ return json.loads(HDFSClient.get_file("{0}/{1}"\
+ .format(hdfs_path,file_name)))
+ else:
+ return {}
+
+"""
+Return a list(dict) with all the data ingested during the time frame provided.
+"""
+def ingest_summary(start_date,end_date):
+
+ db = Configuration.db()
+ is_query = ("""
+ SELECT
+ tdate,total
+ FROM {0}.proxy_ingest_summary
+ WHERE
+ ( y >= {1} and y <= {2}) AND
+ ( m >= {3} and m <= {4}) AND
+ ( d >= {5} and d <= {6})
+ """)\
+ .format(db,start_date.year,end_date.year,start_date.month,end_date.month, start_date.day, end_date.day)
+
+ return ImpalaEngine.execute_query_as_list(is_query)
+
+
+"""
+--------------------------------------------------------------------------
+Reset scored connections.
+--------------------------------------------------------------------------
+"""
+def reset_scored_connections(date):
+
+ proxy_storyboard = "proxy/hive/oa/storyboard"
+ proxy_threat_investigation = "dns_threat_dendro/hive/oa/timeline"
+ proxy_timeline = "proxy/hive/oa/threat_investigation"
+ app_path = Configuration.spot()
+
+ try:
+ # remove parquet files manually to allow the comments update.
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_storyboard,date.year,date.month,date.day) , "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_threat_investigation,date.year,date.month,date.day), "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_timeline,date.year,date.month,date.day), "impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+ return True
+
+ except HdfsError:
+ return False
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/context/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/context/README.md b/spot-oa/context/README.md
index e4869d2..cf65bdc 100644
--- a/spot-oa/context/README.md
+++ b/spot-oa/context/README.md
@@ -1,6 +1,6 @@
This folder must include the following files:
-###**ipranges.csv**
+### **ipranges.csv**
This is a comma separated file, defines a range of IP that should be considered as part of your internal network.
This file will be requested by the following modules:
@@ -14,7 +14,7 @@ Schema with zero-indexed columns:
1.limit_IP : string
2.name : string
-###**iploc.csv**
+### **iploc.csv**
This is a comma separated geolocation database file containing all (or most) known public IP ranges and the details of its location.
This file is required by the following modules:
@@ -46,7 +46,7 @@ Example: 10.180.88.23
Ip to integer = (10 * 16777216) + (180 * 65536) + (88 * 256) + (23) = 179591191.
-###**networkcontext_1.csv**
+### **networkcontext_1.csv**
This is a comma separated file necessary to add more specific context about your network.
This file will be requested by the following modules:
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/README
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/README b/spot-oa/ipython/README
new file mode 100644
index 0000000..f8f2f7f
--- /dev/null
+++ b/spot-oa/ipython/README
@@ -0,0 +1,10 @@
+This is the IPython directory.
+
+For more information on configuring IPython, do:
+
+ipython -h
+
+or to create an empty default profile, populated with default config files:
+
+ipython profile create
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/extensions/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/extensions/__init__.py b/spot-oa/ipython/extensions/__init__.py
new file mode 100644
index 0000000..ecb1860
--- /dev/null
+++ b/spot-oa/ipython/extensions/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/extensions/spot_webapp.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/extensions/spot_webapp.py b/spot-oa/ipython/extensions/spot_webapp.py
new file mode 100644
index 0000000..2f958cb
--- /dev/null
+++ b/spot-oa/ipython/extensions/spot_webapp.py
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+from os import path
+
+sys.path.append(path.dirname(path.dirname(path.dirname(__file__))))
+
+from api.graphql.webapp import load_jupyter_server_extension
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/ipython_notebook_config.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/ipython_notebook_config.py b/spot-oa/ipython/profile_spot/ipython_notebook_config.py
new file mode 100644
index 0000000..9f83d97
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/ipython_notebook_config.py
@@ -0,0 +1,570 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+# Configuration file for ipython-notebook.
+
+c = get_config()
+
+#------------------------------------------------------------------------------
+# NotebookApp configuration
+#------------------------------------------------------------------------------
+
+# NotebookApp will inherit config from: BaseIPythonApplication, Application
+
+# The number of additional ports to try if the specified port is not available.
+# c.NotebookApp.port_retries = 50
+
+# Extra variables to supply to jinja templates when rendering.
+# c.NotebookApp.jinja_template_vars = {}
+
+# The url for MathJax.js.
+# c.NotebookApp.mathjax_url = ''
+
+# Supply extra arguments that will be passed to Jinja environment.
+# c.NotebookApp.jinja_environment_options = {}
+
+# The IP address the notebook server will listen on.
+c.NotebookApp.ip = '0.0.0.0'
+
+# DEPRECATED use base_url
+# c.NotebookApp.base_project_url = '/'
+
+# Create a massive crash report when IPython encounters what may be an internal
+# error. The default is to append a short message to the usual traceback
+# c.NotebookApp.verbose_crash = False
+
+# Python modules to load as notebook server extensions. This is an experimental
+# API, and may change in future releases.
+c.NotebookApp.server_extensions = ['extensions.spot_webapp']
+
+# The random bytes used to secure cookies. By default this is a new random
+# number every time you start the Notebook. Set it to a value in a config file
+# to enable logins to persist across server sessions.
+#
+# Note: Cookie secrets should be kept private, do not share config files with
+# cookie_secret stored in plaintext (you can read the value from a file).
+# c.NotebookApp.cookie_secret = ''
+
+# The default URL to redirect to from `/`
+c.NotebookApp.default_url = '/files/ui/flow/suspicious.html'
+
+# Whether to open in a browser after starting. The specific browser used is
+# platform dependent and determined by the python standard library `webbrowser`
+# module, unless it is overridden using the --browser (NotebookApp.browser)
+# configuration option.
+c.NotebookApp.open_browser = False
+
+# The date format used by logging formatters for %(asctime)s
+# c.NotebookApp.log_datefmt = '%Y-%m-%d %H:%M:%S'
+
+# The port the notebook server will listen on.
+c.NotebookApp.port = 8889
+
+# Whether to overwrite existing config files when copying
+# c.NotebookApp.overwrite = False
+
+# The kernel spec manager class to use. Should be a subclass of
+# `IPython.kernel.kernelspec.KernelSpecManager`.
+#
+# The Api of KernelSpecManager is provisional and might change without warning
+# between this version of IPython and the next stable one.
+# c.NotebookApp.kernel_spec_manager_class = <class 'IPython.kernel.kernelspec.KernelSpecManager'>
+
+# Set the Access-Control-Allow-Origin header
+#
+# Use '*' to allow any origin to access your server.
+#
+# Takes precedence over allow_origin_pat.
+# c.NotebookApp.allow_origin = ''
+
+# The notebook manager class to use.
+# c.NotebookApp.contents_manager_class = <class 'IPython.html.services.contents.filemanager.FileContentsManager'>
+
+# Use a regular expression for the Access-Control-Allow-Origin header
+#
+# Requests from an origin matching the expression will get replies with:
+#
+# Access-Control-Allow-Origin: origin
+#
+# where `origin` is the origin of the request.
+#
+# Ignored if allow_origin is set.
+# c.NotebookApp.allow_origin_pat = ''
+
+# The full path to an SSL/TLS certificate file.
+# c.NotebookApp.certfile = u''
+
+# The logout handler class to use.
+# c.NotebookApp.logout_handler_class = <class 'IPython.html.auth.logout.LogoutHandler'>
+
+# The base URL for the notebook server.
+#
+# Leading and trailing slashes can be omitted, and will automatically be added.
+# c.NotebookApp.base_url = '/'
+
+# The session manager class to use.
+# c.NotebookApp.session_manager_class = <class 'IPython.html.services.sessions.sessionmanager.SessionManager'>
+
+# Supply overrides for the tornado.web.Application that the IPython notebook
+# uses.
+c.NotebookApp.tornado_settings = {
+ 'debug': os.environ.get('SPOT_DEV')=='1'
+}
+
+# The directory to use for notebooks and kernels.
+# c.NotebookApp.notebook_dir = u''
+
+# The kernel manager class to use.
+# c.NotebookApp.kernel_manager_class = <class 'IPython.html.services.kernels.kernelmanager.MappingKernelManager'>
+
+# The file where the cookie secret is stored.
+# c.NotebookApp.cookie_secret_file = u''
+
+# Supply SSL options for the tornado HTTPServer. See the tornado docs for
+# details.
+# c.NotebookApp.ssl_options = {}
+
+#
+# c.NotebookApp.file_to_run = ''
+
+# The IPython profile to use.
+# c.NotebookApp.profile = u'default'
+
+# DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.
+# c.NotebookApp.pylab = 'disabled'
+
+# Whether to enable MathJax for typesetting math/TeX
+#
+# MathJax is the javascript library IPython uses to render math/LaTeX. It is
+# very large, so you may want to disable it if you have a slow internet
+# connection, or for offline use of the notebook.
+#
+# When disabled, equations etc. will appear as their untransformed TeX source.
+c.NotebookApp.enable_mathjax = False
+
+# Reraise exceptions encountered loading server extensions?
+# c.NotebookApp.reraise_server_extension_failures = False
+
+# The base URL for websockets, if it differs from the HTTP server (hint: it
+# almost certainly doesn't).
+#
+# Should be in the form of an HTTP origin: ws[s]://hostname[:port]
+# c.NotebookApp.websocket_url = ''
+
+# The Logging format template
+# c.NotebookApp.log_format = '[%(name)s]%(highlevel)s %(message)s'
+
+# The name of the IPython directory. This directory is used for logging
+# configuration (through profiles), history storage, etc. The default is usually
+# $HOME/.ipython. This option can also be specified through the environment
+# variable IPYTHONDIR.
+# c.NotebookApp.ipython_dir = u''
+
+# The cluster manager class to use.
+# c.NotebookApp.cluster_manager_class = <class 'IPython.html.services.clusters.clustermanager.ClusterManager'>
+
+# Set the log level by value or name.
+# c.NotebookApp.log_level = 30
+
+# Hashed password to use for web authentication.
+#
+# To generate, type in a python/IPython shell:
+#
+# from IPython.lib import passwd; passwd()
+#
+# The string should be of the form type:salt:hashed-password.
+# c.NotebookApp.password = u''
+
+# extra paths to look for Javascript notebook extensions
+# c.NotebookApp.extra_nbextensions_path = []
+
+# Set the Access-Control-Allow-Credentials: true header
+# c.NotebookApp.allow_credentials = False
+
+# Path to an extra config file to load.
+#
+# If specified, load this config file in addition to any other IPython config.
+# c.NotebookApp.extra_config_file = u''
+
+# Extra paths to search for serving static files.
+#
+# This allows adding javascript/css to be available from the notebook server
+# machine, or overriding individual files in the IPython
+# c.NotebookApp.extra_static_paths = []
+
+# The full path to a private key file for usage with SSL/TLS.
+# c.NotebookApp.keyfile = u''
+
+# Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-
+# For headerssent by the upstream reverse proxy. Necessary if the proxy handles
+# SSL
+# c.NotebookApp.trust_xheaders = False
+
+# Extra paths to search for serving jinja templates.
+#
+# Can be used to override templates from IPython.html.templates.
+# c.NotebookApp.extra_template_paths = []
+
+# The config manager class to use
+# c.NotebookApp.config_manager_class = <class 'IPython.html.services.config.manager.ConfigManager'>
+
+# Whether to install the default config files into the profile dir. If a new
+# profile is being created, and IPython contains config files for that profile,
+# then they will be staged into the new directory. Otherwise, default config
+# files will be automatically generated.
+# c.NotebookApp.copy_config_files = False
+
+# The login handler class to use.
+# c.NotebookApp.login_handler_class = <class 'IPython.html.auth.login.LoginHandler'>
+
+# DEPRECATED, use tornado_settings
+# c.NotebookApp.webapp_settings = {}
+
+# Specify what command to use to invoke a web browser when opening the notebook.
+# If not specified, the default browser will be determined by the `webbrowser`
+# standard library module, which allows setting of the BROWSER environment
+# variable to override it.
+# c.NotebookApp.browser = u''
+
+#------------------------------------------------------------------------------
+# KernelManager configuration
+#------------------------------------------------------------------------------
+
+# Manages a single kernel in a subprocess on this host.
+#
+# This version starts kernels with Popen.
+
+# KernelManager will inherit config from: ConnectionFileMixin
+
+# DEPRECATED: Use kernel_name instead.
+#
+# The Popen Command to launch the kernel. Override this if you have a custom
+# kernel. If kernel_cmd is specified in a configuration file, IPython does not
+# pass any arguments to the kernel, because it cannot make any assumptions about
+# the arguments that the kernel understands. In particular, this means that the
+# kernel does not receive the option --debug if it given on the IPython command
+# line.
+# c.KernelManager.kernel_cmd = []
+
+# Should we autorestart the kernel if it dies.
+# c.KernelManager.autorestart = False
+
+# set the stdin (ROUTER) port [default: random]
+# c.KernelManager.stdin_port = 0
+
+# Set the kernel's IP address [default localhost]. If the IP address is
+# something other than localhost, then Consoles on other machines will be able
+# to connect to the Kernel, so be careful!
+# c.KernelManager.ip = u''
+
+# JSON file in which to store connection info [default: kernel-<pid>.json]
+#
+# This file will contain the IP, ports, and authentication key needed to connect
+# clients to this kernel. By default, this file will be created in the security
+# dir of the current profile, but can be specified by absolute path.
+# c.KernelManager.connection_file = ''
+
+# set the control (ROUTER) port [default: random]
+# c.KernelManager.control_port = 0
+
+# set the heartbeat port [default: random]
+# c.KernelManager.hb_port = 0
+
+# set the shell (ROUTER) port [default: random]
+# c.KernelManager.shell_port = 0
+
+#
+# c.KernelManager.transport = 'tcp'
+
+# set the iopub (PUB) port [default: random]
+# c.KernelManager.iopub_port = 0
+
+#------------------------------------------------------------------------------
+# ProfileDir configuration
+#------------------------------------------------------------------------------
+
+# An object to manage the profile directory and its resources.
+#
+# The profile directory is used by all IPython applications, to manage
+# configuration, logging and security.
+#
+# This object knows how to find, create and manage these directories. This
+# should be used by any code that wants to handle profiles.
+
+# Set the profile location directly. This overrides the logic used by the
+# `profile` option.
+# c.ProfileDir.location = u''
+
+#------------------------------------------------------------------------------
+# Session configuration
+#------------------------------------------------------------------------------
+
+# Object for handling serialization and sending of messages.
+#
+# The Session object handles building messages and sending them with ZMQ sockets
+# or ZMQStream objects. Objects can communicate with each other over the
+# network via Session objects, and only need to work with the dict-based IPython
+# message spec. The Session will handle serialization/deserialization, security,
+# and metadata.
+#
+# Sessions support configurable serialization via packer/unpacker traits, and
+# signing with HMAC digests via the key/keyfile traits.
+#
+# Parameters ----------
+#
+# debug : bool
+# whether to trigger extra debugging statements
+# packer/unpacker : str : 'json', 'pickle' or import_string
+# importstrings for methods to serialize message parts. If just
+# 'json' or 'pickle', predefined JSON and pickle packers will be used.
+# Otherwise, the entire importstring must be used.
+#
+# The functions must accept at least valid JSON input, and output *bytes*.
+#
+# For example, to use msgpack:
+# packer = 'msgpack.packb', unpacker='msgpack.unpackb'
+# pack/unpack : callables
+# You can also set the pack/unpack callables for serialization directly.
+# session : bytes
+# the ID of this Session object. The default is to generate a new UUID.
+# username : unicode
+# username added to message headers. The default is to ask the OS.
+# key : bytes
+# The key used to initialize an HMAC signature. If unset, messages
+# will not be signed or checked.
+# keyfile : filepath
+# The file containing a key. If this is set, `key` will be initialized
+# to the contents of the file.
+
+# Username for the Session. Default is your system username.
+# c.Session.username = u'diego'
+
+# The name of the unpacker for unserializing messages. Only used with custom
+# functions for `packer`.
+# c.Session.unpacker = 'json'
+
+# Threshold (in bytes) beyond which a buffer should be sent without copying.
+# c.Session.copy_threshold = 65536
+
+# The name of the packer for serializing messages. Should be one of 'json',
+# 'pickle', or an import name for a custom callable serializer.
+# c.Session.packer = 'json'
+
+# The maximum number of digests to remember.
+#
+# The digest history will be culled when it exceeds this value.
+# c.Session.digest_history_size = 65536
+
+# The UUID identifying this session.
+# c.Session.session = u''
+
+# The digest scheme used to construct the message signatures. Must have the form
+# 'hmac-HASH'.
+# c.Session.signature_scheme = 'hmac-sha256'
+
+# execution key, for signing messages.
+# c.Session.key = ''
+
+# Debug output in the Session
+# c.Session.debug = False
+
+# The maximum number of items for a container to be introspected for custom
+# serialization. Containers larger than this are pickled outright.
+# c.Session.item_threshold = 64
+
+# path to file containing execution key.
+# c.Session.keyfile = ''
+
+# Threshold (in bytes) beyond which an object's buffer should be extracted to
+# avoid pickling.
+# c.Session.buffer_threshold = 1024
+
+# Metadata dictionary, which serves as the default top-level metadata dict for
+# each message.
+# c.Session.metadata = {}
+
+#------------------------------------------------------------------------------
+# MappingKernelManager configuration
+#------------------------------------------------------------------------------
+
+# A KernelManager that handles notebook mapping and HTTP error handling
+
+# MappingKernelManager will inherit config from: MultiKernelManager
+
+# The name of the default kernel to start
+# c.MappingKernelManager.default_kernel_name = 'python2'
+
+#
+# c.MappingKernelManager.root_dir = u''
+
+# The kernel manager class. This is configurable to allow subclassing of the
+# KernelManager for customized behavior.
+# c.MappingKernelManager.kernel_manager_class = 'IPython.kernel.ioloop.IOLoopKernelManager'
+
+#------------------------------------------------------------------------------
+# ContentsManager configuration
+#------------------------------------------------------------------------------
+
+# Base class for serving files and directories.
+#
+# This serves any text or binary file, as well as directories, with special
+# handling for JSON notebook documents.
+#
+# Most APIs take a path argument, which is always an API-style unicode path, and
+# always refers to a directory.
+#
+# - unicode, not url-escaped
+# - '/'-separated
+# - leading and trailing '/' will be stripped
+# - if unspecified, path defaults to '',
+# indicating the root path.
+
+# The base name used when creating untitled files.
+# c.ContentsManager.untitled_file = 'untitled'
+
+# Python callable or importstring thereof
+#
+# To be called on a contents model prior to save.
+#
+# This can be used to process the structure, such as removing notebook outputs
+# or other side effects that should not be saved.
+#
+# It will be called as (all arguments passed by keyword)::
+#
+# hook(path=path, model=model, contents_manager=self)
+#
+# - model: the model to be saved. Includes file contents.
+# Modifying this dict will affect the file that is stored.
+# - path: the API path of the save destination
+# - contents_manager: this ContentsManager instance
+# c.ContentsManager.pre_save_hook = None
+
+#
+# c.ContentsManager.checkpoints_class = <class 'IPython.html.services.contents.checkpoints.Checkpoints'>
+
+# Glob patterns to hide in file and directory listings.
+# c.ContentsManager.hide_globs = [u'__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dylib', '*~']
+
+# The base name used when creating untitled notebooks.
+# c.ContentsManager.untitled_notebook = 'Untitled'
+
+# The base name used when creating untitled directories.
+# c.ContentsManager.untitled_directory = 'Untitled Folder'
+
+#
+# c.ContentsManager.checkpoints = None
+
+#
+# c.ContentsManager.checkpoints_kwargs = {}
+
+#------------------------------------------------------------------------------
+# FileContentsManager configuration
+#------------------------------------------------------------------------------
+
+# FileContentsManager will inherit config from: ContentsManager
+
+#
+# c.FileContentsManager.root_dir = u''
+
+# The base name used when creating untitled files.
+# c.FileContentsManager.untitled_file = 'untitled'
+
+# Python callable or importstring thereof
+#
+# to be called on the path of a file just saved.
+#
+# This can be used to process the file on disk, such as converting the notebook
+# to a script or HTML via nbconvert.
+#
+# It will be called as (all arguments passed by keyword)::
+#
+# hook(os_path=os_path, model=model, contents_manager=instance)
+#
+# - path: the filesystem path to the file just written - model: the model
+# representing the file - contents_manager: this ContentsManager instance
+# c.FileContentsManager.post_save_hook = None
+
+# Python callable or importstring thereof
+#
+# To be called on a contents model prior to save.
+#
+# This can be used to process the structure, such as removing notebook outputs
+# or other side effects that should not be saved.
+#
+# It will be called as (all arguments passed by keyword)::
+#
+# hook(path=path, model=model, contents_manager=self)
+#
+# - model: the model to be saved. Includes file contents.
+# Modifying this dict will affect the file that is stored.
+# - path: the API path of the save destination
+# - contents_manager: this ContentsManager instance
+# c.FileContentsManager.pre_save_hook = None
+
+#
+# c.FileContentsManager.checkpoints_class = <class 'IPython.html.services.contents.checkpoints.Checkpoints'>
+
+# Glob patterns to hide in file and directory listings.
+# c.FileContentsManager.hide_globs = [u'__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dylib', '*~']
+
+# The base name used when creating untitled notebooks.
+# c.FileContentsManager.untitled_notebook = 'Untitled'
+
+# The base name used when creating untitled directories.
+# c.FileContentsManager.untitled_directory = 'Untitled Folder'
+
+#
+# c.FileContentsManager.checkpoints = None
+
+#
+# c.FileContentsManager.checkpoints_kwargs = {}
+
+# DEPRECATED, use post_save_hook
+# c.FileContentsManager.save_script = False
+
+#------------------------------------------------------------------------------
+# NotebookNotary configuration
+#------------------------------------------------------------------------------
+
+# A class for computing and verifying notebook signatures.
+
+# The number of notebook signatures to cache. When the number of signatures
+# exceeds this value, the oldest 25% of signatures will be culled.
+# c.NotebookNotary.cache_size = 65535
+
+# The secret key with which notebooks are signed.
+# c.NotebookNotary.secret = ''
+
+# The sqlite file in which to store notebook signatures. By default, this will
+# be in your IPython profile. You can set it to ':memory:' to disable sqlite
+# writing to the filesystem.
+# c.NotebookNotary.db_file = u''
+
+# The file where the secret key is stored.
+# c.NotebookNotary.secret_file = u''
+
+# The hashing algorithm used to sign notebooks.
+# c.NotebookNotary.algorithm = 'sha256'
+
+#------------------------------------------------------------------------------
+# KernelSpecManager configuration
+#------------------------------------------------------------------------------
+
+# Whitelist of allowed kernel names.
+#
+# By default, all installed kernels are allowed.
+# c.KernelSpecManager.whitelist = set([])
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/startup/README
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/startup/README b/spot-oa/ipython/profile_spot/startup/README
new file mode 100644
index 0000000..61d4700
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/startup/README
@@ -0,0 +1,11 @@
+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+ 00-first.py
+ 50-middle.py
+ 99-last.ipy
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/startup/graphql.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/startup/graphql.py b/spot-oa/ipython/profile_spot/startup/graphql.py
new file mode 100644
index 0000000..9aea6b6
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/startup/graphql.py
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import IPython
+import json
+import urllib3
+import os
+
+class GraphQLClient:
+ def __init__(self, url=None):
+ self.url = url or 'http://localhost:{}/graphql'.format(self.get_nbserver_info()['port'])
+ self.variables = None
+
+ def get_nbserver_info(self):
+ profile_loc = IPython.config.get_config()['ProfileDir']['location']
+ nbserver_pid = os.getppid()
+ nbserver_file = os.path.join(profile_loc, 'security', 'nbserver-{}.json'.format(nbserver_pid))
+
+ try:
+ return json.load(open(nbserver_file))
+ except:
+ return {}
+
+ def set_query(self, query):
+ self.query = query
+
+ def set_variables(self, variables):
+ self.variables = variables
+
+ def send_query(self):
+ assert(self.url is not None)
+ assert(type(self.url) is str)
+ assert(self.query is not None)
+ assert(type(self.query) is str)
+
+ data = {
+ 'query': self.query
+ }
+
+ if self.variables is not None and type(self.variables) is dict:
+ data['variables'] = self.variables
+
+ encoded_data = json.dumps(data).encode('utf-8')
+
+ http = urllib3.PoolManager()
+
+ response = http.request(
+ 'POST',
+ self.url,
+ body=encoded_data,
+ headers={
+ 'Accept': 'application/json',
+ 'Content-type': 'application/json'
+ }
+ )
+
+ try:
+ return json.loads(response.data.decode('utf-8'))
+ except:
+ return {
+ 'errors': [
+ {
+ 'status': response.status,
+ 'message': 'Failed to contact GraphQL endpoint. Is "{}" the correct URL?'.format(self.url)
+ }
+ ]
+ }
+
+ @classmethod
+ def request(cls, query, variables=None, url=None):
+ client = cls(url)
+
+ client.set_query(query)
+ if variables is not None:
+ client.set_variables(variables)
+
+ return client.send_query()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif b/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif
new file mode 100644
index 0000000..7dc5c7a
Binary files /dev/null and b/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/static/custom/custom.css
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/static/custom/custom.css b/spot-oa/ipython/profile_spot/static/custom/custom.css
new file mode 100755
index 0000000..b8bef4a
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/static/custom/custom.css
@@ -0,0 +1,163 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+.spot.spot_easy_mode {
+ background-color: white;
+}
+
+.spot.spot_easy_mode:not(.spot_easy_mode_loading) #site {
+ height: 100% !important;
+}
+
+.spot.spot_easy_mode:not(.spot_easy_mode_loading) #notebook {
+ padding: 0 !important;
+ box-shadow: none;
+ -webkit-box-shadow: none;
+}
+
+.spot.spot_easy_mode #header,
+.spot.spot_easy_mode #pager,
+.spot.spot_easy_mode #tooltip,
+.spot.spot_easy_mode #notebook > .end_space,
+.spot.spot_easy_mode .cell:not(.code_cell),
+.spot.spot_easy_mode .cell.code_cell > .input,
+.spot.spot_easy_mode .cell.code_cell > .widget-area > .prompt {
+ position: absolute !important;
+ top: -1000000000px;
+ left: -1000000000px;
+}
+
+.spot.spot_easy_mode.spot_easy_mode_loading .cell.code_cell {
+ position: absolute !important;
+ top: -1000000000px;
+ left: -1000000000px;
+}
+
+.spot.spot_easy_mode #notebook-container {
+ -webkit-box-shadow: none;
+ box-shadow: none;
+}
+
+.spot.spot_easy_mode .cell.code_cell {
+ border-color: transparent !important;
+ padding: 0 !important;
+}
+
+/*****************************************
+ ** EASY MODE SPINNER **
+ *****************************************/
+.spot #spot_easy_mode_loader {
+ display: none;
+}
+
+.spot.spot_easy_mode #spot_easy_mode_loader {
+ display: block;
+ position: fixed;
+ top: 10px;
+ width: 100%;
+ text-align: center;
+ margin-top: 50px;
+}
+
+.spot.spot_easy_mode #spot_easy_mode_loader_spinner
+{
+ display: inline-block;
+ width: 16px;
+ height: 11px;
+ margin-left: 5px;
+ background-image: url(./ajax-loader.gif);
+}
+
+.spot.spot_easy_mode:not(.spot_easy_mode_loading) #spot_easy_mode_loader
+{
+ right: 25px;
+ margin-top: 0px;
+ width: auto;
+}
+
+.spot.spot_ninja_mode #spot_easy_mode_loader
+{
+ display: none;
+}
+
+/*****************************************
+ ** Spot TOOLTIP STYLING **
+ *****************************************/
+.spot .spot-tooltip .tooltip-inner {
+ word-wrap: break-word;
+}
+
+.spot .spot-text-wrapper.overflown {
+ color: red;
+}
+
+.spot .spot-text-wrapper {
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ width: 12em;
+}
+
+.spot .spot-text-wrapper.spot-text-sm {
+ width: 6em;
+}
+
+.spot .spot-text-wrapper.spot-text-lg {
+ width: 18em;
+}
+
+.spot .spot-text-wrapper.spot-text-xlg {
+ width: 30em;
+}
+
+/* Classes for the threat investigation notebooks*/
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table{
+ margin:10px 10px 20px 0px;
+ font-size:12px;
+}
+
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td, th,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td, th{
+ border: solid 1px #CCC;
+ padding: 3px;
+ background-color: #FFF;
+ line-height:14px;
+}
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td{
+ height:25px;
+ max-width: 180px;
+ width:100px;
+}
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table th,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table th{
+
+ background-color: #0071C5;
+ color: #FFF;
+ text-align: center;
+}
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area > .widget-subarea,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area > .widget-subarea
+{
+ width: 90%;
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/static/custom/custom.js
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/static/custom/custom.js b/spot-oa/ipython/profile_spot/static/custom/custom.js
new file mode 100755
index 0000000..93579dc
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/static/custom/custom.js
@@ -0,0 +1,351 @@
+// #
+// # Licensed to the Apache Software Foundation (ASF) under one or more
+// # contributor license agreements. See the NOTICE file distributed with
+// # this work for additional information regarding copyright ownership.
+// # The ASF licenses this file to You under the Apache License, Version 2.0
+// # (the "License"); you may not use this file except in compliance with
+// # the License. You may obtain a copy of the License at
+// #
+// # http://www.apache.org/licenses/LICENSE-2.0
+// #
+// # Unless required by applicable law or agreed to in writing, software
+// # distributed under the License is distributed on an "AS IS" BASIS,
+// # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// # See the License for the specific language governing permissions and
+// # limitations under the License.
+// #
+/**
+ * Trigger reload event on parent document
+ */
+function reloadParentData()
+{
+ window.parent.iframeReloadHook && window.parent.iframeReloadHook();
+}
+
+require(['jquery', 'bootstrap'], function($, bootstrap)
+{
+ function retry(query) {
+ var regex = new RegExp(query + '=(\\d)');
+
+ var matcher = regex.exec(window.location.hash);
+
+ var attempt = matcher ? ++matcher[1] : 2;
+
+ // Set hash
+ window.location.hash = '#' + query + '=' + attempt;
+ window.location.reload();
+ }
+
+ // jQuery must be available for easy mode to woark properly
+ if ($===undefined) {
+ if (/#spot_jquery_retry=[015-9]/.test(window.location.hash)) {
+ alert('Notebook was not able to load jQuery after 5 attempts. Please try again later, this is a known issue.');
+ console.warn('Bootstrap\'s tooltip was not loaded.');
+ }
+ else {
+ confirm('Notebook failed to load jQuery, easy mode would not work properly without it. Would you you like to try again?')
+ && retry('spot_jquery_retry');
+ }
+
+ return;
+ }
+
+ if ($.fn.tooltip===undefined) {
+ if (/#spot_bootstrap_retry=\d/.test(window.location.hash)) {
+ alert('Notebook was not able to load bootstrap\'s tooltip plugin. You can still work withuot this feature.');
+ } else {
+ confirm('Notebook was not able to load bootstrap\'s tooltip plugin. You can still work withuot this feature. Would you like to try again?')
+ && retry('spot_bootstrap_retry');
+ }
+ }
+
+ var easyMode = {
+ stage: 0,
+ KERNEL_READY: 1,
+ NOTEBOOK_READY: 2,
+ DOM_READY: 4,
+ ENV_READY: 7,
+ building: false,
+ present: false,
+ ready: false,
+ cells: {
+ total: 0,
+ execution_queue: []
+ }
+ }
+
+ /**
+ * Show EasyMode
+ *
+ * Hide everything but IPython form
+ */
+ function showEasyMode()
+ {
+ $(document.body).addClass('spot_easy_mode').removeClass('spot_ninja_mode');
+ }
+
+ /**
+ * Hide Easy Mode
+ *
+ * Show full IPython Notebook
+ */
+ function hideEasyMode()
+ {
+ $(document.body).addClass('spot_ninja_mode').removeClass('spot_easy_mode');
+ }
+
+ function insertProgressIndicator()
+ {
+ $(document.body).append(
+ '<div id="spot_easy_mode_loader">' +
+ '<span id="spot_easy_mode_loader_details"></span>' +
+ '<span id="spot_easy_mode_loader_spinner"></span>' +
+ '</div>'
+ );
+ }
+
+ /**
+ * Remove progress indicator
+ */
+ function removeProgressIndicator()
+ {
+ $(document.body).removeClass('spot_easy_mode_loading');
+
+ $('#spot_easy_mode_loader').remove();
+ }
+
+ /**
+ * Updates progress indicator's details
+ */
+ function updateProgressIndicator(content)
+ {
+ $('#spot_easy_mode_loader_details').html(content);
+ }
+
+ /**
+ * Show a progress indicator to users
+ */
+ function showBuildingUiIndicator()
+ {
+ $(document.body).addClass('spot_easy_mode_loading');
+
+ insertProgressIndicator();
+
+ updateProgressIndicator(
+ 'Building UI <span id="spot_easy_mode_loader_progress">0</span>%'
+ );
+ }
+
+ /**
+ * Update progress indicator
+ */
+ function updateBuildingUiIndicator()
+ {
+ var p;
+
+ p = (easyMode.cells.total-easyMode.cells.execution_queue.length) * 100 / easyMode.cells.total;
+
+ $('#spot_easy_mode_loader_progress').text(Math.floor(p));
+ }
+
+ function easyModeBootStrap (IPython)
+ {
+ if (easyMode.stage!=easyMode.ENV_READY) return;
+
+ // 1 Build widgets
+ easyMode.building = true;
+
+ console.info('Spot: Building easy mode...');
+
+ // 2 Create an execution queue to display progress
+ easyMode.cells.execution_queue = [];
+
+ easyMode.cells.total = 0;
+ IPython.notebook.get_cells().forEach(function (cell)
+ {
+ if (cell.cell_type==='code')
+ {
+ easyMode.cells.total++;
+ }
+ });
+
+ // Add an extra cell to show progress when requesting execution
+ easyMode.cells.total++;
+
+ // 3 Execute all cells ( Generate UI )
+ IPython.notebook.execute_all_cells();
+
+ updateBuildingUiIndicator();
+ }
+
+ function isEasyModeAvailable()
+ {
+ return window.parent!=window;
+ }
+
+ function isEasyModeEnabled() {
+ return /showEasyMode/.test(window.location.hash);
+ }
+
+ function isNinjaModeEnabled() {
+ return /showNinjaMode/.test(window.location.hash);
+ }
+
+ if (isEasyModeAvailable()) {
+ // Add spot CSS classes (easymode enabled by default)
+ $(document.body).addClass('spot').addClass('spot_easy_mode').addClass('spot_easy_mode_loading');
+
+ // Listen for URL's hash changes
+ $(window).on('hashchange', function ()
+ {
+ if (isEasyModeEnabled()) showEasyMode();
+ else if (isNinjaModeEnabled()) hideEasyMode();
+ });
+
+ $(function () {
+ // Show progress indicator
+ showBuildingUiIndicator();
+ });
+ }
+
+ // Enable spot tooltip text wrapper
+ $(function () {
+ $('body').tooltip({
+ selector: '.spot-text-wrapper[data-toggle]',
+ container: 'body',
+ template: '<div class="spot-tooltip tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',
+ title: function () {
+ return $(this).html();
+ },
+ html: true
+ });
+
+ $('body').on('show.bs.tooltip', '.spot-text-wrapper', function () {
+ return this.clientWidth !== this.scrollWidth || this.clientHeight !== this.scrollHeight;
+ });
+ });
+
+ /**
+ * The following code enables toggle from normal user mode (wizard) and ninja node (notebook UI)
+ */
+ require(['base/js/namespace', 'base/js/events'], function(IPython, events)
+ {
+ // Do nothing when running stand alone
+ if (!isEasyModeAvailable()) return;
+
+ // We are running inside and iframe from Spot. Let's have some fun!
+
+ // Let Notebook be aware it is running on an iframe
+ IPython._target = '_self';
+
+ events.on('kernel_busy.Kernel', function ()
+ {
+ // Skip this event while building UI
+ if (!easyMode.ready) return;
+
+ $('#notebook button.btn:not([disabled])').addClass('spotDisabled').attr('disabled', 'disabled');
+
+ insertProgressIndicator();
+ });
+
+ events.on('kernel_idle.Kernel', function ()
+ {
+ // Skip this event while building UI
+ if (!easyMode.ready) return;
+
+ removeProgressIndicator();
+
+ $('#notebook button.btn.spotDisabled').removeClass('spotDisabled').removeAttr('disabled');
+ });
+
+ events.on('kernel_ready.Kernel', function ()
+ {
+ console.info('Spot: Kernel is ready');
+
+ easyMode.stage |= easyMode.KERNEL_READY;
+
+ easyModeBootStrap(IPython);
+ });
+
+ events.on('notebook_loaded.Notebook', function ()
+ {
+ console.info('Spot: Notebook loaded');
+
+ easyMode.stage |= easyMode.NOTEBOOK_READY;
+
+ easyModeBootStrap(IPython);
+ });
+
+ events.on('shell_reply.Kernel', function (evt, data)
+ {
+ var reply, cell, cellIdx;
+
+ reply = data.reply;
+ cell = easyMode.cells.execution_queue.shift();
+
+ console.log('Last execution status: ' + reply.content.status);
+
+ if ((easyMode.building || easyMode.ready) && reply.content.status==='error')
+ {
+ // First error found
+ easyMode.building = false;
+ easyMode.ready = false;
+ isEasyModeEnabled() && alert('Ooops some code failed. Please go to ipython notebook mode and manually fix the error.');
+ $(document.body).removeClass('spot');
+ removeProgressIndicator();
+ hideEasyMode();
+ // Select and scroll to first cell with errors
+ cellIdx = IPython.notebook.find_cell_index(cell);
+ IPython.notebook.scroll_to_cell(cellIdx);
+ IPython.notebook.select(cellIdx);
+ }
+
+ if (!easyMode.building)
+ {
+ return;
+ }
+
+ if (easyMode.cells.execution_queue.length===0)
+ {
+ console.info('Spot: Cell execution has finished');
+
+ easyMode.ready = true;
+ easyMode.building = false;
+
+ removeProgressIndicator();
+ isEasyModeEnabled() && showEasyMode();
+ }
+ else
+ {
+ updateBuildingUiIndicator();
+ }
+ });
+
+ events.on('execute.CodeCell', function (ev, obj)
+ {
+ var cell;
+
+ if (!easyMode.building && !easyMode.ready) return;
+
+ cell = obj.cell;
+
+ easyMode.cells.execution_queue.push(cell);
+
+ console.info('Spot: Cell execution requested: ' + easyMode.cells.execution_queue.length + ' of ' + easyMode.cells.total);
+
+ cell.clear_output(false);
+ // There seems to be a bug on IPython sometimes cells with widgets dont get cleared
+ // Workaround:
+ cell.output_area.clear_output(false, true);
+ });
+
+ $(function ()
+ {
+ console.info('Spot: DOM is ready');
+
+ easyMode.stage |= easyMode.DOM_READY;
+
+ easyModeBootStrap(IPython);
+ });
+ });
+});
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/INSTALL.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/INSTALL.md b/spot-oa/oa/INSTALL.md
index 342775e..f870a55 100644
--- a/spot-oa/oa/INSTALL.md
+++ b/spot-oa/oa/INSTALL.md
@@ -32,7 +32,7 @@ More information about oa/proxy in [README.md](/spot-oa/oa/proxy)
More information about oa/components in [README.md](/spot-oa/oa/components)
-##Operational Analytics prerequisites
+## Operational Analytics prerequisites
In order to execute this process there are a few prerequisites:
@@ -49,11 +49,11 @@ In order to execute this process there are a few prerequisites:
If users want to implement their own machine learning piece to detect suspicious connections they need to refer
to each data type module to know more about input format and schema.
4. spot-setup project installed. Spot-setup project contains scripts to install hive database and the main configuration
- file for Open Network Insights project.
+ file.
-##Operational Analytics installation and usage
-####Installation
+## Operational Analytics installation and usage
+#### Installation
OA installation consists of the configuration of extra modules or components and creation of a set of files.
Depending on the data type that is going to be processed some components are required and other components are not.
@@ -97,7 +97,7 @@ In order to execute this process there are a few prerequisites:
configure all of them in case new data types are analyzed in the future.
For more details about how to configure each component go to [spot-oa/oa/components/README.md](/spot-oa/oa/components/README.md).
- ####Usage
+ #### Usage
OA process is triggered with the execution of start_oa.py. This Python script will execute the OA process
for only one data type (Flow, DNS or Proxy) at a time. If users need to process multiple data types at the same time, multiple
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/README.md b/spot-oa/oa/components/README.md
index 3353493..ee0e286 100644
--- a/spot-oa/oa/components/README.md
+++ b/spot-oa/oa/components/README.md
@@ -26,7 +26,7 @@ This document will explain the necessary steps to configure the spot-oa componen
| geoloc -> Module to assign geolocation to every IP
-###Data
+### Data
_Data source module._
This module needs to be configured correctly to avoid errors during the spot-oa execution. Here you need to select the correct database engine to obtain the correct results while creating additional details files.
@@ -61,7 +61,7 @@ Example:
}
-###Reputation
+### Reputation
_Reputation check module._
This module is called during spot-oa execution to check the reputation for any given IP, DNS name or URI (depending on the pipeline). The reputation module makes use of two third-party services, McAfee GTI and Facebook ThreatExchange.
@@ -124,7 +124,7 @@ To add a different reputation service, you can read all about it [here](reputati
- app_id: App id to connect to ThreatExchange service.
- app_secret: App secret to connect to ThreatExchange service.
-###IANA
+### IANA
_Internet Assigned Numbers Authority codes translation module._
**Configuration**
@@ -146,7 +146,7 @@ default location, your configuration file should look like this:
}
-###Network Context (nc)
+### Network Context (nc)
_Network Context module._
**Pre-requisites**
@@ -175,7 +175,7 @@ configuration file should look like this:
}
-###Geoloc
+### Geoloc
_Geolocation module._
This is an optional functionality you can enable / disable depending on your preferences.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/__init__.py b/spot-oa/oa/components/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/__init__.py
+++ b/spot-oa/oa/components/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/data/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/data/__init__.py b/spot-oa/oa/components/data/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/data/__init__.py
+++ b/spot-oa/oa/components/data/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/geoloc/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/geoloc/__init__.py b/spot-oa/oa/components/geoloc/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/geoloc/__init__.py
+++ b/spot-oa/oa/components/geoloc/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/iana/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/iana/__init__.py b/spot-oa/oa/components/iana/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/iana/__init__.py
+++ b/spot-oa/oa/components/iana/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/iana/iana_transform.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/iana/iana_transform.py b/spot-oa/oa/components/iana/iana_transform.py
index 97a1c28..a49c8f9 100644
--- a/spot-oa/oa/components/iana/iana_transform.py
+++ b/spot-oa/oa/components/iana/iana_transform.py
@@ -82,20 +82,20 @@ class IanaTransform(object):
if key in self._qclass_dict:
return self._qclass_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
if column == COL_QTYPE:
if key in self._qtype_dict:
return self._qtype_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
if column == COL_RCODE:
if key in self._rcode_dict:
return self._rcode_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
if column == COL_PRESP:
if key in self._http_rcode_dict:
return self._http_rcode_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/nc/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/nc/__init__.py b/spot-oa/oa/components/nc/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/nc/__init__.py
+++ b/spot-oa/oa/components/nc/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/README.md b/spot-oa/oa/components/reputation/README.md
index 60f3cbd..67cf918 100644
--- a/spot-oa/oa/components/reputation/README.md
+++ b/spot-oa/oa/components/reputation/README.md
@@ -1,4 +1,4 @@
-###Reputation
+### Reputation
This section describes the functionality of the current reputation service modules and how you can implement your own.
It's possible to add new reputation services by implementing a new sub-module, to do that developers should follow
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/__init__.py b/spot-oa/oa/components/reputation/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/reputation/__init__.py
+++ b/spot-oa/oa/components/reputation/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/fb/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/fb/__init__.py b/spot-oa/oa/components/reputation/fb/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/reputation/fb/__init__.py
+++ b/spot-oa/oa/components/reputation/fb/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/fb/fb.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/fb/fb.py b/spot-oa/oa/components/reputation/fb/fb.py
index e9d997c..b33ec7d 100644
--- a/spot-oa/oa/components/reputation/fb/fb.py
+++ b/spot-oa/oa/components/reputation/fb/fb.py
@@ -92,7 +92,7 @@ class Reputation(object):
str_response = urllib2.urlopen(request).read()
response = json.loads(str_response)
except urllib2.HTTPError as e:
- self._logger.error("Error calling ThreatExchange in module fb: " + e.message)
+ self._logger.info("Error calling ThreatExchange in module fb: " + e.message)
reputation_dict[name] = self._get_reputation_label('UNKNOWN')
return reputation_dict
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/gti/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/gti/__init__.py b/spot-oa/oa/components/reputation/gti/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/reputation/gti/__init__.py
+++ b/spot-oa/oa/components/reputation/gti/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/gti/gti.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/gti/gti.py b/spot-oa/oa/components/reputation/gti/gti.py
index 0e2147f..473ccbd 100644
--- a/spot-oa/oa/components/reputation/gti/gti.py
+++ b/spot-oa/oa/components/reputation/gti/gti.py
@@ -88,6 +88,7 @@ class Reputation(object):
responses += self._call_gti(command, len(queries))
ip_counter = 0
+ category_name_group = ""
for query_resp in responses:
if self.AFLAG_KEY in query_resp or self.REP_KEY not in query_resp :
reputation_dict[values[ip_counter]] = self._get_reputation_label(self.DEFAULT_REP)
[18/18] incubator-spot git commit: syncing with master
Posted by na...@apache.org.
syncing with master
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/fbcfa683
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/fbcfa683
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/fbcfa683
Branch: refs/heads/SPOT-181_ODM
Commit: fbcfa6835a99ca4d8499a9dc24c2c8ce12d4284f
Parents: 2bf696e
Author: natedogs911 <na...@gmail.com>
Authored: Tue Sep 26 14:36:08 2017 -0700
Committer: natedogs911 <na...@gmail.com>
Committed: Tue Sep 26 14:36:08 2017 -0700
----------------------------------------------------------------------
LICENSE | 97 +
README.md | 2 +-
dev/release/.rat-excludes | 11 +
dev/release/README.md | 474 +
docs/label-bug-enhancement.bmp | Bin 29390 -> 0 bytes
docs/label-discussion.bmp | Bin 13074 -> 0 bytes
docs/label-issue-type.bmp | Bin 161250 -> 0 bytes
docs/label-repos.bmp | Bin 245838 -> 0 bytes
docs/label-tracking.bmp | Bin 181638 -> 0 bytes
docs/labels-design-tracking.bmp | Bin 181638 -> 0 bytes
docs/oni-approach.png | Bin 58589 -> 0 bytes
docs/oni-guy.png | Bin 34146 -> 0 bytes
docs/oni-guy_big.png | Bin 90430 -> 0 bytes
docs/oni-logo.png | Bin 16768 -> 0 bytes
docs/open-data-model/open-data-model.md | 892 -
spot-ingest/common/__init__.py | 16 +
spot-ingest/pipelines/__init__.py | 17 +-
spot-ingest/pipelines/dns/__init__.py | 16 +
.../pipelines/dns/load_dns_avro_parquet.hql | 15 +
spot-ingest/pipelines/flow/__init__.py | 16 +
.../pipelines/flow/load_flow_avro_parquet.hql | 15 +
spot-ingest/pipelines/proxy/__init__.py | 16 +
spot-ingest/pipelines/proxy/bluecoat.py | 2 +-
spot-ml/DATA_SAMPLE.md | 57 +
spot-ml/INSTALL.md | 2 +-
spot-ml/ML_OPS.md | 8 +-
spot-ml/README.md | 17 +-
spot-ml/SPOT-ML-JAR.md | 4 +-
spot-ml/SUSPICIOUS_CONNECTS_SCHEMA.md | 4 +-
spot-ml/build.sbt | 18 +-
spot-ml/ml_ops.sh | 7 +-
spot-ml/ml_test.sh | 6 +-
spot-ml/project/build.properties | 1 +
spot-ml/project/plugins.sbt | 17 +
.../org/apache/spot/SuspiciousConnects.scala | 18 +-
.../dns/DNSSuspiciousConnectsAnalysis.scala | 62 +-
.../org/apache/spot/dns/model/DNSFeedback.scala | 14 +-
.../dns/model/DNSSuspiciousConnectsModel.scala | 28 -
.../FlowSuspiciousConnectsAnalysis.scala | 51 +-
.../spot/netflow/model/FlowFeedback.scala | 21 +-
.../model/FlowSuspiciousConnectsModel.scala | 2 +-
.../org/apache/spot/proxy/ProxyFeedback.scala | 28 +-
.../proxy/ProxySuspiciousConnectsAnalysis.scala | 60 +-
.../proxy/ProxySuspiciousConnectsModel.scala | 18 +-
.../org/apache/spot/utilities/TopDomains.scala | 3 +-
.../utilities/data/InputOutputDataHandler.scala | 33 +-
.../utilities/data/validation/InputSchema.scala | 51 +
spot-ml/src/test/resources/log4j.properties | 18 +-
.../SuspiciousConnectsArgumentParserTest.scala | 17 +
.../dns/DNSSuspiciousConnectsAnalysisTest.scala | 56 +-
.../dns/DNSSuspiciousConnectsModelTest.scala | 17 +
.../org/apache/spot/netflow/FlowRecord.scala | 35 +
.../FlowSuspiciousConnectsAnalysisTest.scala | 56 +-
.../model/FlowSuspiciousConnectsModelTest.scala | 17 +
.../ProxySuspiciousConnectsAnalysisTest.scala | 87 +-
.../FloatPointPrecisionUtility32Test.scala | 17 +
.../spot/utilities/FloatingPointUtility64.scala | 17 +
.../data/validation/InputSchemaTest.scala | 53 +
spot-ml/top-1m.csv | 1000000 ---------------
spot-oa/README.md | 10 +-
spot-oa/__init__.py | 16 +
spot-oa/api/__init__.py | 16 +
spot-oa/api/graphql/README.md | 13 +
spot-oa/api/graphql/__init__.py | 18 +
spot-oa/api/graphql/common.py | 142 +
spot-oa/api/graphql/dns/__init__.py | 18 +
spot-oa/api/graphql/dns/mutation.py | 182 +
spot-oa/api/graphql/dns/query.py | 519 +
spot-oa/api/graphql/flow/__init__.py | 18 +
spot-oa/api/graphql/flow/mutation.py | 218 +
spot-oa/api/graphql/flow/query.py | 691 +
spot-oa/api/graphql/proxy/__init__.py | 18 +
spot-oa/api/graphql/proxy/mutation.py | 221 +
spot-oa/api/graphql/proxy/query.py | 582 +
spot-oa/api/graphql/schema.py | 69 +
spot-oa/api/graphql/webapp.py | 52 +
spot-oa/api/resources/README.md | 49 +
spot-oa/api/resources/__init__.py | 16 +
spot-oa/api/resources/configurator.py | 59 +
spot-oa/api/resources/dns.py | 395 +
spot-oa/api/resources/flow.py | 958 +
spot-oa/api/resources/hdfs_client.py | 76 +
spot-oa/api/resources/impala_engine.py | 50 +
spot-oa/api/resources/proxy.py | 467 +
spot-oa/context/README.md | 6 +-
spot-oa/ipython/README | 10 +
spot-oa/ipython/extensions/__init__.py | 16 +
spot-oa/ipython/extensions/spot_webapp.py | 22 +
.../profile_spot/ipython_notebook_config.py | 570 +
spot-oa/ipython/profile_spot/startup/README | 11 +
spot-oa/ipython/profile_spot/startup/graphql.py | 90 +
.../profile_spot/static/custom/ajax-loader.gif | Bin 0 -> 723 bytes
.../profile_spot/static/custom/custom.css | 163 +
.../profile_spot/static/custom/custom.js | 351 +
spot-oa/oa/INSTALL.md | 10 +-
spot-oa/oa/components/README.md | 10 +-
spot-oa/oa/components/__init__.py | 16 +
spot-oa/oa/components/data/__init__.py | 16 +
spot-oa/oa/components/geoloc/__init__.py | 16 +
spot-oa/oa/components/iana/__init__.py | 16 +
spot-oa/oa/components/iana/iana_transform.py | 8 +-
spot-oa/oa/components/nc/__init__.py | 16 +
spot-oa/oa/components/reputation/README.md | 2 +-
spot-oa/oa/components/reputation/__init__.py | 16 +
spot-oa/oa/components/reputation/fb/__init__.py | 16 +
spot-oa/oa/components/reputation/fb/fb.py | 2 +-
.../oa/components/reputation/gti/__init__.py | 16 +
spot-oa/oa/components/reputation/gti/gti.py | 1 +
spot-oa/oa/dns/README.md | 131 +-
spot-oa/oa/dns/__init__.py | 16 +
spot-oa/oa/dns/dns_conf.json | 36 +-
spot-oa/oa/dns/dns_oa.py | 317 +-
.../ipynb_templates/Advanced_Mode_master.ipynb | 292 +
spot-oa/oa/dns/ipynb_templates/EdgeNotebook.md | 76 -
.../Edge_Investigation_master.ipynb | 269 -
.../dns/ipynb_templates/ThreatInvestigation.md | 79 +-
.../Threat_Investigation_master.ipynb | 233 +-
spot-oa/oa/flow/README.md | 134 +-
spot-oa/oa/flow/__init__.py | 16 +
spot-oa/oa/flow/flow_conf.json | 27 +-
spot-oa/oa/flow/flow_oa.py | 269 +-
.../ipynb_templates/Advanced_Mode_master.ipynb | 292 +
spot-oa/oa/flow/ipynb_templates/EdgeNotebook.md | 97 -
.../Edge_Investigation_master.ipynb | 407 -
.../flow/ipynb_templates/ThreatInvestigation.md | 191 +-
.../Threat_Investigation_master.ipynb | 681 +-
spot-oa/oa/proxy/README.md | 139 +-
spot-oa/oa/proxy/__init__.py | 16 +
.../ipynb_templates/Advanced_Mode_master.ipynb | 296 +
.../oa/proxy/ipynb_templates/EdgeNotebook.md | 74 -
.../Edge_Investigation_master.ipynb | 229 -
.../ipynb_templates/ThreatInvestigation.md | 88 +-
.../Threat_Investigation_master.ipynb | 336 +-
spot-oa/oa/proxy/proxy_conf.json | 10 +-
spot-oa/oa/proxy/proxy_oa.py | 248 +-
spot-oa/oa/start_oa.py | 31 +-
spot-oa/oa/utils.py | 12 +
spot-oa/requirements.txt | 17 +-
spot-oa/runIpython.sh | 11 +-
spot-oa/ui/README.md | 27 +-
spot-oa/ui/css/bootstrap-spot.min.css | 16 +
spot-oa/ui/css/dendrogram.css | 16 +
spot-oa/ui/css/main.css | 71 +
spot-oa/ui/dns/ipython_notebook.html | 108 +
.../components/DetailsDendrogramPanel.react.js | 22 +-
.../ui/dns/js/components/DetailsPanel.react.js | 17 +-
.../js/components/DetailsTablePanel.react.js | 21 +-
.../ui/dns/js/components/FilterInput.react.js | 17 +-
.../IncidentProgressionPanel.react.js | 21 +-
.../dns/js/components/NetworkViewPanel.react.js | 19 +-
.../ui/dns/js/components/ScoreNotebook.react.js | 206 +
.../dns/js/components/SuspiciousPanel.react.js | 39 +-
spot-oa/ui/dns/js/constants/DnsConstants.js | 12 -
spot-oa/ui/dns/js/notebooks.js | 72 +
spot-oa/ui/dns/js/stores/CommentsStore.js | 86 +-
spot-oa/ui/dns/js/stores/DendrogramStore.js | 103 +-
spot-oa/ui/dns/js/stores/DetailsStore.js | 129 +-
.../dns/js/stores/IncidentProgressionStore.js | 154 +-
spot-oa/ui/dns/js/stores/NotebookStore.js | 107 +
spot-oa/ui/dns/js/stores/SuspiciousStore.js | 352 +-
spot-oa/ui/dns/js/storyboard.js | 23 +-
spot-oa/ui/dns/js/suspicious.js | 51 +-
spot-oa/ui/dns/js/threat-investigation.js | 23 +-
spot-oa/ui/dns/package.json | 6 +-
spot-oa/ui/dns/storyboard.html | 58 +-
spot-oa/ui/dns/suspicious.html | 61 +-
spot-oa/ui/dns/threat-investigation.html | 58 +-
spot-oa/ui/flow/ipython_notebook.html | 108 +
.../js/components/DetailsChordsPanel.react.js | 17 +-
.../ui/flow/js/components/DetailsPanel.react.js | 17 +-
.../js/components/DetailsTablePanel.react.js | 17 +-
.../ui/flow/js/components/FilterInput.react.js | 17 +-
.../flow/js/components/GlobeViewPanel.react.js | 17 +-
.../js/components/ImpactAnalysisPanel.react.js | 17 +-
.../IncidentProgressionPanel.react.js | 17 +-
.../js/components/NetworkViewPanel.react.js | 20 +-
.../flow/js/components/ScoreNotebook.react.js | 221 +
.../flow/js/components/SuspiciousPanel.react.js | 20 +-
.../flow/js/components/TimelinePanel.react.js | 17 +-
.../ui/flow/js/constants/NetflowConstants.js | 27 +-
spot-oa/ui/flow/js/lib/topojson.min.js | 2 -
spot-oa/ui/flow/js/notebooks.js | 72 +
spot-oa/ui/flow/js/stores/ChordsDiagramStore.js | 106 +-
spot-oa/ui/flow/js/stores/CommentsStore.js | 77 +-
spot-oa/ui/flow/js/stores/DetailsStore.js | 159 +-
spot-oa/ui/flow/js/stores/GlobeViewStore.js | 119 +-
.../ui/flow/js/stores/ImpactAnalysisStore.js | 97 +-
.../flow/js/stores/IncidentProgressionStore.js | 87 +-
spot-oa/ui/flow/js/stores/NotebookStore.js | 107 +
spot-oa/ui/flow/js/stores/SuspiciousStore.js | 306 +-
spot-oa/ui/flow/js/stores/TimelineStore.js | 99 +-
spot-oa/ui/flow/js/storyboard.js | 23 +-
spot-oa/ui/flow/js/suspicious.js | 51 +-
spot-oa/ui/flow/js/threat-investigation.js | 23 +-
spot-oa/ui/flow/package.json | 6 +-
spot-oa/ui/flow/storyboard.html | 58 +-
spot-oa/ui/flow/suspicious.html | 62 +-
spot-oa/ui/flow/threat-investigation.html | 58 +-
spot-oa/ui/images/favicon/browserconfig.xml | 14 +
spot-oa/ui/ingest-summary.html | 183 -
spot-oa/ui/ingest/ingest-summary.html | 132 +
spot-oa/ui/ipython/custom/ajax-loader.gif | Bin 723 -> 0 bytes
spot-oa/ui/ipython/custom/custom.css | 145 -
spot-oa/ui/ipython/custom/custom.js | 337 -
spot-oa/ui/js/actions/EdInActions.js | 34 +-
spot-oa/ui/js/actions/InSumActions.js | 17 +-
spot-oa/ui/js/actions/SpotActions.js | 23 +-
spot-oa/ui/js/actions/StoryboardActions.js | 17 +-
.../ui/js/actions/__tests__/EdInActions.test.js | 15 +-
.../ui/js/actions/__tests__/SpotActions.test.js | 16 +-
.../actions/__tests__/StoryboardActions.test.js | 16 +-
spot-oa/ui/js/components/ChartMixin.react.js | 17 +-
.../js/components/ContentLoaderMixin.react.js | 17 +-
spot-oa/ui/js/components/DateInput.react.js | 17 +-
.../ui/js/components/DendrogramMixin.react.js | 17 +-
.../ui/js/components/DetailsGridMixin.react.js | 18 +-
.../ExecutiveThreatBriefingPanel.react.js | 17 +-
.../ui/js/components/GridPanelMixin.react.js | 17 +-
.../js/components/IPythonNotebookPanel.react.js | 25 +-
.../js/components/IngestSummaryPanel.react.js | 19 +-
spot-oa/ui/js/components/OptionPicker.react.js | 17 +-
spot-oa/ui/js/components/Panel.react.js | 28 +-
spot-oa/ui/js/components/PanelRow.react.js | 20 +-
.../components/PolloNetworkViewMixin.react.js | 19 +-
.../js/components/SuspiciousGridMixin.react.js | 17 +-
spot-oa/ui/js/components/TimelineMixin.react.js | 17 +-
.../components/scoring/ButtonsScoring.react.js | 48 +
.../scoring/FilterSelectInput.react.js | 45 +
.../js/components/scoring/RatingInput.react.js | 42 +
.../js/components/scoring/ScoreMessage.react.js | 38 +
.../scoring/SearchGlobalInput.react.js | 49 +
.../js/components/scoring/SelectInput.react.js | 46 +
spot-oa/ui/js/constants/SpotConstants.js | 26 +-
spot-oa/ui/js/cubehelix/cubehelix.js | 17 +-
spot-oa/ui/js/dispatchers/SpotDispatcher.js | 17 +-
spot-oa/ui/js/ingest-summary.js | 35 +-
spot-oa/ui/js/menu/components/MainMenu.react.js | 73 +
spot-oa/ui/js/menu/menu.js | 52 +
spot-oa/ui/js/menu/stores/MainMenuStore.js | 57 +
spot-oa/ui/js/stores/GraphQLStore.js | 89 +
spot-oa/ui/js/stores/IngestSummaryStore.js | 236 +-
spot-oa/ui/js/stores/JsonStore.js | 17 +-
spot-oa/ui/js/stores/ObservableGraphQLStore.js | 55 +
.../stores/ObservableWithHeadersGraphQLStore.js | 31 +
spot-oa/ui/js/stores/RestStore.js | 17 +-
spot-oa/ui/js/stores/SpotStore.js | 17 +-
.../ui/js/stores/__tests__/JsonStore.test.js | 16 +-
.../ui/js/stores/__tests__/RestStore.test.js | 16 +-
.../ui/js/stores/__tests__/SpotStore.test.js | 16 +-
spot-oa/ui/js/tooltip.js | 17 +-
spot-oa/ui/js/utils/CategoryLayout.js | 17 +-
spot-oa/ui/js/utils/DateUtils.js | 17 +-
spot-oa/ui/js/utils/SpotUtils.js | 52 +-
.../js/utils/__tests__/CategoryLayout.test.js | 16 +-
spot-oa/ui/js/utils/__tests__/DateUtils.test.js | 16 +-
spot-oa/ui/js/utils/__tests__/SpotUtils.test.js | 16 +-
spot-oa/ui/package.json | 5 +-
spot-oa/ui/proxy/ipython_notebook.html | 108 +
.../proxy/js/components/DetailsPanel.react.js | 20 +-
.../ui/proxy/js/components/FilterInput.react.js | 17 +-
.../IncidentProgressionPanel.react.js | 17 +-
.../js/components/NetworkViewPanel.react.js | 23 +-
.../proxy/js/components/ScoreNotebook.react.js | 199 +
.../js/components/SuspiciousPanel.react.js | 20 +-
.../proxy/js/components/TimelinePanel.react.js | 18 +-
spot-oa/ui/proxy/js/constants/ProxyConstants.js | 12 -
spot-oa/ui/proxy/js/notebooks.js | 72 +
spot-oa/ui/proxy/js/stores/CommentsStore.js | 77 +-
spot-oa/ui/proxy/js/stores/DetailsStore.js | 140 +-
.../proxy/js/stores/IncidentProgressionStore.js | 83 +-
spot-oa/ui/proxy/js/stores/NotebookStore.js | 107 +
spot-oa/ui/proxy/js/stores/SuspiciousStore.js | 310 +-
spot-oa/ui/proxy/js/stores/TimelineStore.js | 108 +-
spot-oa/ui/proxy/js/storyboard.js | 23 +-
spot-oa/ui/proxy/js/suspicious.js | 45 +-
spot-oa/ui/proxy/js/threat-investigation.js | 23 +-
spot-oa/ui/proxy/package.json | 6 +-
spot-oa/ui/proxy/storyboard.html | 58 +-
spot-oa/ui/proxy/suspicious.html | 61 +-
spot-oa/ui/proxy/threat-investigation.html | 58 +-
spot-setup/README.md | 60 +-
spot-setup/create_dns_avro_parquet.hql | 37 -
spot-setup/create_dns_parquet.hql | 163 +
spot-setup/create_flow_avro_parquet.hql | 72 -
spot-setup/create_flow_parquet.hql | 195 +
spot-setup/create_proxy_avro_parquet.hql | 76 -
spot-setup/create_proxy_parquet.hql | 177 +
spot-setup/hdfs_setup.sh | 47 +-
spot-setup/migrate_to_spot_1_0.py | 102 +
spot-setup/migration/README.md | 71 +
spot-setup/migration/__init__.py | 16 +
.../migration/create_dns_migration_tables.hql | 113 +
.../migration/create_flow_migration_tables.hql | 126 +
.../migration/create_proxy_migration_tables.hql | 127 +
.../migration/drop_dns_migration_tables.hql | 23 +
.../migration/drop_flow_migration_tables.hql | 23 +
.../migration/drop_proxy_migration_tables.hql | 23 +
spot-setup/migration/migrate_old_dns_data.py | 248 +
spot-setup/migration/migrate_old_flow_data.py | 240 +
spot-setup/migration/migrate_old_proxy_data.py | 236 +
spot-setup/migration/spot_conf_migration.py | 135 +
spot-setup/migration/utilities.py | 89 +
spot-setup/spot.conf | 29 +-
303 files changed, 17548 insertions(+), 1006718 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
index 8f71f43..7e097d1 100644
--- a/LICENSE
+++ b/LICENSE
@@ -200,3 +200,100 @@
See the License for the specific language governing permissions and
limitations under the License.
+
+=======================================================================
+Apache Spot Subcomponents:
+
+The Apache Spot (incubating) project contains subcomponents with separate
+copyright notices and license terms. Your use of the source code for these
+subcomponents is subject to the terms and conditions of the following
+licenses.
+
+========================================================================
+For topojson (spot-oa/ui/flow/js/lib/topojson.min.js):
+========================================================================
+
+Copyright (c) 2012-2016, Michael Bostock All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+ • Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+
+ • Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in
+ the documentation and/or other materials provided with the distribution.
+
+ • The name Michael Bostock may not be used to endorse or promote
+ products derived from this software without specific prior written
+ permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS”
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+========================================================================
+For chordMapper (spot-oa/ui/flow/js/lib/ChordMapper.js):
+========================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2014 Steve Hall
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+
+========================================================================
+For D3's World 110m (spot-oa/ui/flow/world-110m.json):
+========================================================================
+
+Copyright (c) 2014, Michael Bostock
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+ list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+ this list of conditions and the following disclaimer in the documentation
+ and/or other materials provided with the distribution.
+
+* The name Michael Bostock may not be used to endorse or promote products
+ derived from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL MICHAEL BOSTOCK BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
+EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 8b4ee42..13888ea 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
# **Apache Spot (Incubating)**
-Apache Spot is open source software for leveraging insights from flow and packet analysis. It helps enterprises and service providers gain insight on their compute environments through transparency of service delivery and identification of potential security threats or attacks happening among resources operating at cloud scale.
+Apache Spot is open source software for leveraging insights from flow and packet analysis. It helps enterprises and service providers gain insight on their computing environments through transparency of service delivery and identification of potential security threats or attacks happening among resources operating at cloud scale.
While current threat intelligence tools help, identifying unknown threats and attacks remains a challenge. Apache Spot provides tools to accelerate companies’ ability to expose suspicious connections and previously unseen attacks using flow and packet analysis technologies.
<br><br>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/dev/release/.rat-excludes
----------------------------------------------------------------------
diff --git a/dev/release/.rat-excludes b/dev/release/.rat-excludes
new file mode 100644
index 0000000..2441af7
--- /dev/null
+++ b/dev/release/.rat-excludes
@@ -0,0 +1,11 @@
+.*md
+.*txt
+.gitignore
+.gitmodules
+.*png
+.*json
+.*csv
+.*less
+.*ipynb
+.babelrc
+topojson.min.js
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/dev/release/README.md
----------------------------------------------------------------------
diff --git a/dev/release/README.md b/dev/release/README.md
new file mode 100644
index 0000000..fb9a18c
--- /dev/null
+++ b/dev/release/README.md
@@ -0,0 +1,474 @@
+# **Apache Spot (Incubating) Release Process**
+
+## **Overview**
+
+This document describes the Release Process to perform the official Apache Spot (Incubating) release following the [Apache Software Foundation Release Policy](http://incubator.apache.org/guides/releasemanagement.html#best-practice).
+
+### **Requirements**
+
+As a Release Manager (RM), you should create a code signing key to sign the release artifacts following this [guide](http://www.apache.org/dev/openpgp.html#generate-key).
+
+Public key should be detached and added in to the KEYS file in Spot Repo under: https://dist.apache.org/repos/dist/dev/incubator/spot/KEYS
+
+### **Policy documents**
+
+Frequently asked questions for making Apache releases are available on [Releases FAQ page](http://www.apache.org/legal/release-policy.html#releases).
+
+The Release Manager must go through the policy document to understand all the tasks and responsibilities of running a release.
+
+### **Give a heads up**
+
+The release manager should create an EPIC in Jira and then setup a timeline for release branch point. The time for the day the EPIC is created to the release branch point must be at least two weeks in order to give the community a chance to prioritize and commit any last minute features and issues they would like to see in the upcoming release.
+
+The release manager should then send the pointer to the EPIC along with the tentative timeline (Code Freeze) for branch point to the user and developer lists. Any work identified as release related that needs to be completed should be added as a subtask of the umbrella issue to allow users to see the overall release progress in one place.
+
+ To: dev@spot.apache.org
+
+ Subject: Work on Spot <your release name> (Incubating) Release has started + Code Freeze Date
+
+
+ We are starting the process to prepare for Spot <your release name> (Incubating) release. I have opened JIRA $jira to cover the features included in this release.
+
+ If you have any JIRA in progress and would like to include in this release, please follow the process to do so. Code Freeze for final integration will be on $code_freeze_date.
+
+ Feel free to comment on the JIRA if you have any comments/suggestions.
+
+ Thanks,
+ <Release Manager Name>
+
+### **Sanitize Jira**
+
+Before a release is done, make sure that any issues that are fixed have their fix version setup correctly. If the release number is not listed in the "fix version" field, as RM create a ticket for Infrastructure asking to create the value for the "fix version" field.
+
+Once the value is created by Infrastructure team, run the following JIRA query to see which resolved issues do not have their fix version set up correctly:
+
+project = spot and resolution = fixed and fixVersion is empty
+
+The result of the above query should be empty. If some issues do show up in this query that have been fixed since the last release, please bulk-edit them to set the fix version to '1.0'.
+
+You can also run the following query to make sure that the issues fixed for the to-be-released version look accurate:
+
+project = spot and resolution = fixed and fixVersion = '1.0'
+
+### **Monitor active issues**
+
+It is important that between the time that the umbrella issue is filed to the time when the release branch is created, no experimental or potentially destabilizing work is checked into the trunk. While it is acceptable to introduce major changes, they must be thoroughly reviewed and have good test coverage to ensure that the release branch does not start of being unstable.
+
+If necessary the RM can discuss if certain issues should be fixed on the trunk in this time, and if so what is the gating criteria for accepting them.
+
+### **Pull Request Validation**
+
+All the features that will be included in the release EPIC needs to have a proper Pull Request (PR) created following this guide. And needs three +1 votes which at least one of them must be from the QA Team.
+
+Development must be done in to a Topic Branch or Master Branch, depending on the scope of the release.
+
+Once the PR has votes, then the PPMC developer must merge the PR and the owner of that PR should close it properly in Github (in case it does not close automatically).
+
+If Development is performed in Topic Branch, then Topic Branch should be merged in to master branch once development is done.
+
+### **Create the Release Candidate**
+
+The Release candidate branch will be created from Master branch once all pull requests form the EPIC are merged to a topic branch or directly to master branch before code freeze date.
+
+#### **Branch your release:**
+
+* git checkout -b `<your release name>` `<commit sha1>`
+
+push to origin:
+
+* git push origin `<your release name>`
+
+#### **Tag your Branch release:**
+Apply signed tag on release branch that will indicate where the release candidate was generated.
+
+Example:
+* git tag -u `<GPG KEY ID>` --sign `<your release name>`-incubating -m "Apache Spot `<your release name>` (Incubating)" `<SHA of HEAD of branch>`
+
+### **Run RAT**
+Apache Rat is a release audit tool, focused on licenses. Used to improve accuracy and efficiency when checking releases for licenses.
+
+Download RAT:
+* wget http://apache.claz.org//creadur/apache-rat-0.12/apache-rat-0.12-bin.tar.gz
+
+Decompress the code:
+* tar -zxvf apache-rat-0.12-bin.tar.gz
+* cd apache-rat-0.12
+
+Now lets create the file to exclude the known extensions and files:
+* vi or nano .rat-excludes
+
+Add the following exclude list:
+
+ .*md
+ .*txt
+ .gitignore
+ .gitmodules
+ .*png
+ .*json
+ .*csvss
+ .*less
+ .*ipynb
+ .babelrc
+ topojson.min.js
+
+Save the File.
+
+Or download the .rat-excludes files from: https://github.com/apache/incubator-spot/blob/master/dev/release/.rat-excludes
+
+Run the rat tool as following.
+* java -jar apache-rat-0.12.jar -E /path/to/project/.rat-excludes -d /path/to/project/ > `<to output file>`.txt
+
+If you have rat in the same directory as the Spot Code you can verify as:
+* java -jar apache-rat-0.12.jar -E .rat-excludes -d ../apache-spot-1.0-incubating > apache-spot-1.0-incubating-rat-results.txt
+
+If RAT find problems in the licenses please fix the licence and run RAT again developers must fix their code and submit changes into the release branch, once there are no more findings. Upload RAT Results into subversion dev incubator repo for Spot of the release
+
+#### **Make a tarball and gzip:**
+* git archive -o ../apache-spot-`<your release name>`-incubating.tar --prefix=apache-spot-`<your release name>`-incubating/ `<your tag/branch name>`
+* gzip ../apache-spot-`<your release name>`-incubating.tar
+
+Example:
+
+ $ git archive -o ../apache-spot-1.0-incubating.tar --prefix=apache-spot-1.0-incubating/ 1.0-incubating
+ $ gzip ../apache-spot-1.0-incubating.tar
+
+#### **Prepare MD5, SHA512 and ASC files from the source tarball:**
+
+* md5 apache-spot-`<your release name>`-incubating.tar.gz > apache-spot-`<your release name>`-incubating.tar.gz.md5
+* shasum -a 512 apache-spot-`<your release name>`-incubating.tar.gz > apache-spot-`<your release name>`-incubating.tar.gz.sha512
+* gpg2 --detach-sign -a apache-spot-`<your release name>`-incubating.tar.gz
+
+Example:
+
+ $ md5 apache-spot-1.0-incubating..tar.gz > apache-spot-1.0-incubating.tar.gz.md5
+ $ shasum -a 512 apache-spot-1.0-incubating.tar.gz > apache-spot-1.0-incubating.tar.gz.sha512
+ $ gpg2 --detach-sign -a apache-spot-1.0-incubating..tar.gz
+
+
+#### **Retrieve the subversion dev incubator repo for Spot**
+
+Example:
+* svn checkout https://dist.apache.org/repos/dist/dev/incubator/spot/ --username=`<your apache user>`
+
+Create a local folder for the release (e.g. 1.0-incubating) in svn.
+* svn mkdir -m "Creating Spot `<release number>` dir" https://dist.apache.org/repos/dist/dev/incubator/spot/`<release number>` --username=`<your apache user>`
+
+Example:
+
+ svn mkdir -m "Creating Spot 1.0-incubating dir" https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incuabting --username=`<your apache user>`
+
+
+Move the files into the release folder on local disk.
+* svn add `<release folder>`
+
+Example:
+
+ svn add 1.0-incubating/
+
+Commit artifacts:
+* svn commit -m '`<custom message>`' --username=`<your apache user id>`
+
+Example:
+
+ svn commit -m 'adding spot 1.0-incubating candidate release artifacts' --username=`<your apache user id>`
+
+## **Validate the Build**
+
+Download the tarball.
+
+* http://spot.apache.org/download
+
+Decompress the tarball. Instruction:
+
+ tar -zxvf apache-spot-1.0-incubating.tar.gz
+
+Change directory. Instruction:
+
+ cd apache-spot-1.0-incubator
+
+Apache Spot (incubating) is composed of more than one module or sub-projects. Since some of them are Python or Javascript code, they don’t need compilation.
+
+For more instructions about how to install each module please read below instructions.
+
+ You should see the content of the folder:
+
+ spotadmin-mac01:apache-spot-1.0-incubating spotadmin$ ls -la
+ total 72
+ drwxr-xr-x 14 spotadmin staff 476 Jul 24 16:45 .
+ drwxr-xr-x 7 spotadmin staff 238 Aug 4 09:32 ..
+ -rw-r--r-- 1 spotadmin staff 20 Jul 24 16:45 .gitignore
+ -rw-r--r-- 1 spotadmin staff 0 Jul 24 16:45 .gitmodules
+ -rw-r--r-- 1 spotadmin staff 560 Jul 24 16:45 DISCLAIMER
+ -rw-r--r-- 1 spotadmin staff 11918 Jul 24 16:45 LICENSE
+ -rw-r--r-- 1 spotadmin staff 1493 Jul 24 16:45 LICENSE-topojson.txt
+ -rw-r--r-- 1 spotadmin staff 159 Jul 24 16:45 NOTICE
+ -rw-r--r-- 1 spotadmin staff 6761 Jul 24 16:45 README.md
+ drwxr-xr-x 3 spotadmin staff 102 Jul 24 16:45 docs
+ drwxr-xr-x 10 spotadmin staff 340 Jul 24 16:45 spot-ingest
+ drwxr-xr-x 13 spotadmin staff 442 Jul 24 16:45 spot-ml
+ drwxr-xr-x 11 spotadmin staff 374 Jul 24 16:45 spot-oa
+ drwxr-xr-x 10 spotadmin staff 340 Jul 24 16:45 spot-setup
+
+Decompressed tarball content should be the same with the content located in:
+
+* [https://github.com/apache/incubator-spot/tree/v1.0-incubating](https://github.com/apache/incubator-spot/tree/v1.0-incubating)
+
+To install the properly component please follow this guide:
+
+* [http://spot.apache.org/doc/#installation](http://spot.apache.org/doc/#installation)
+
+Spot Ingest, Spot Setup, Spot OA and Spot UI have specific requirements to install manually.
+
+* [http://spot.apache.org/doc/#configuration](http://spot.apache.org/doc/#configuration)
+* [http://spot.apache.org/doc/#ingest](http://spot.apache.org/doc/#ingest)
+* [http://spot.apache.org/doc/#oa](http://spot.apache.org/doc/#oa)
+* [http://spot.apache.org/doc/#ui](http://spot.apache.org/doc/#ui)
+
+Spot ML is the only component to build the binary files using sbt assembly commands. Please follows these instructions.
+
+* [http://spot.apache.org/doc/#ml](http://spot.apache.org/doc/#ml)
+
+
+## **Running the Vote**
+
+As per the Apache Incubator release [guidelines](http://incubator.apache.org/policy/incubation.html#Releases), all releases for incubating projects must go through a two-step voting process. First, release voting must successfully pass within the Apache Spot (Incubating) community via the dev@spot.incubator.apache.org mail list. Then, release voting must successfully pass within the Apache Incubator PMC via the general@incubator.apache.org mail list.
+
+### **Call for Spot Community Vote**
+
+Call for Vote in spot dev community sending an email to dev list.
+
+For example,
+
+ To: dev@spot.apache.org
+ Subject: [VOTE] Release Apache Spot 1.0-incubating
+
+ Hi All,
+
+ This is the vote for Apache Spot 1.0 (incubating) release.
+
+ The vote will run for at least 72 hours and will close on July 27,2017.
+
+ Release Notes (Jira generated):
+ https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12320637&version=12340668
+
+ Git Branch and Tag for the release:
+ https://github.com/apache/incubator-spot/tree/branch-1.0
+ https://github.com/apache/incubator-spot/tree/v1.0-incubating
+
+ Source code for the release:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz
+
+ Source release verification:
+ PGP Signature:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz.asc
+
+ MD5/SHA512 Hash:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz.md5
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz.sha512
+
+ RAT license Verification:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating-rat-results.txt
+
+ Keys to verify the signature of the release artifact are available at:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/KEYS
+
+ The artifact(s) have been signed with Key : 06B82CAEDB5B280349E75D5533CD9431141E946C
+
+ Download the release candidate and evaluate the necessary items including checking hashes, signatures, source code and test.
+ Please vote accordingly:
+ [ ] +1 approve
+ [ ] +0 no opinion
+ [ ] -1 disapprove (and reason why)
+
+ =================
+ DISCLAIMER
+
+ Apache Spot (incubating) is an effort undergoing incubation at the Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC.
+ Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects.
+
+ While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that the project has yet to be fully endorsed by the ASF.
+ =================
+
+ --
+ Best Regards!
+ -----------------------------------
+ <Release Manager Name>
+ http://spot.apache.org/
+ -----------------------------------
+
+Allow the community to vote and any -1 vote with comments please fix ASAP and send findings to the thread until the required votes are reached by the PPMC members of Spot.
+
+Send a following email with the results that should include the counting votes and thread of the results listed in http://lists.apache.org
+
+ To: dev@spot.apache.org
+ Subject: [RESULT][VOTE] Release Apache Spot 1.0-incubating
+
+ Hi All,
+
+ The voting process for the Release Apache Spot 1.0-incubating is now closed with the following and positive results:
+
+ [10] Binding Votes
+ [1] Non-binding
+
+ Thread of the voting email with responses can be found here:
+
+ https://lists.apache.org/thread.html/69dfe2626c7b803e2a3f26e4d348be8d1941003f0e8166fb8e0e9679@%3Cdev.spot.apache.org%3E
+
+ The next step will be sending the release artifacts for voting at the Incubator General list to get the IPMC approval to officially declare the release.
+
+
+ Thanks
+ --
+ Best Regards!
+ -----------------------------------
+ <Release Manager Name>
+ http://spot.apache.org/
+ -----------------------------------
+
+### **Call for Incubator PMC Vote**
+
+The second voting is the most important since it is required to get three +1 (Binding) vote from the IPMC members from the Incubator General list to declare an official release.
+
+Send the vote to the general Incubator list and include the voting results from the dev list as evidence.
+
+ To: general@incubator.apache.org
+ Subject: [VOTE] Release Apache Spot 1.0-incubating
+
+ Dear IPMC team,
+
+ This is the vote for Apache Spot 1.0 (incubating) release. This is the first release of Spot.
+
+ Apache Spot (Incubating) is open source software for leveraging insights from flow and packet analysis. It helps enterprises and service providers gain insight on their network environments through transparency of service delivery and identification of potential security threats or attacks happening among resources operating at cloud scale. While current threat intelligence tools help, identifying unknown threats and attacks remains a challenge. Apache Spot provides tools to accelerate companies’ ability to expose suspicious connections and previously unseen attacks using flow and packet analysis technologies.
+
+ The PPMC Vote Threads can be found here:
+ https://lists.apache.org/thread.html/69dfe2626c7b803e2a3f26e4d348be8d1941003f0e8166fb8e0e9679@%3Cdev.spot.apache.org%3E
+
+
+ The PPMC vote results can be found here:
+ https://lists.apache.org/thread.html/a88ef44e0dcda9013781eeca363ad9b3439f6c34a698c6eaa50fb314@%3Cdev.spot.apache.org%3E
+
+
+ Release Notes (Jira generated):
+ https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12320637&version=12340668
+
+ Git Branch and Tag for the release:
+ https://github.com/apache/incubator-spot/tree/branch-1.0
+ https://github.com/apache/incubator-spot/tree/v1.0-incubating
+
+ Source code for the release:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz
+
+ Source release verification:
+ http://nolamarketing.com/client/apache-spot/download/
+
+ PGP Signature:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz.asc
+
+ MD5/SHA512 Hash:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz.md5
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating.tar.gz.sha512
+
+ RAT license Verification:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incubating/apache-spot-1.0-incubating-rat-results.txt
+
+ Keys to verify the signature of the release artifact are available at:
+ https://dist.apache.org/repos/dist/dev/incubator/spot/KEYS
+
+ The artifact(s) have been signed with Key : 06B82CAEDB5B280349E75D5533CD9431141E946C
+
+ Download the release candidate and evaluate the necessary items.
+
+ Please vote accordingly:
+ [ ] +1, approve as the official Apache Spot 1.0-incubating release
+ [ ] -1, do not accept as the official as the official Apache Spot 1.0-incubating release because...
+
+ The vote will run for at least 72 hours or until necessary number of votes are reached.
+
+ =================
+ DISCLAIMER
+
+ Apache Spot (incubating) is an effort undergoing incubation at the Apache Software Foundation (ASF), sponsored by the Apache Incubator PMC.
+ Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects.
+
+ While incubation status is not necessarily a reflection of the completeness or stability of the code, it does indicate that project has yet to be fully endorsed by the ASF.
+ =================
+
+ --
+ Best Regards!
+ -----------------------------------
+ <Release Manager Name>
+ http://spot.apache.org/
+ -----------------------------------
+
+Monitor the voting thread and make sure you have the required votes, if feedback is provides fix ASAP and send the updates to the voting thread so voting reaches the required votes.
+
+Once we have the three +1 (Binding) votes then send the following email with the results:
+
+
+ To: general@incubator.apache.org
+ Subject: [RESULT][VOTE] Release Apache Spot 1.0-incubating
+
+ Hi All,
+
+ The voting process for the Release Apache Spot 1.0-incubating is now closed with the following and positive results:
+
+ [3] Binding Votes
+ [1] Non-binding
+
+ Thread of the voting email with responses can be found here:
+
+ https://lists.apache.org/thread.html/32d7c93fe66cc256ed12a5b8f91b57b1d0d659b9012c8f4f13c11191@%3Cgeneral.incubator.apache.org%3E
+
+
+ I will prepare the artifacts to officially release Apache Spot 1.0-incubating.
+
+
+ Thanks
+ --
+ Best Regards!
+ -----------------------------------
+ <Release Manager Name>
+ http://spot.apache.org/
+ -----------------------------------
+
+
+Moving the Artifacts to release stage in SVN using the following command.
+* svn move -m "`<comment>`" `<Directory Origin>` `<Directory Destination>` --username=`<your apache user id>`
+
+Example:
+
+ svn move -m "Moving Apache Spot 1.0-incubating release artifacts to release stage" /* https://dist.apache.org/repos/dist/dev/incubator/spot/1.0-incuabting https://dist.apache.org/repos/dist/release/incubator/spot/1.0-incuabting --username=`<your apache user id>`
+
+Allow 24 hours before updating the webpage and announcing the new Release in Apache Spot (Incubating) webpage. http://nolamarketing.com/client/apache-spot/download/
+
+### **Update WebPages**
+
+You need to update the Spot webpages to reflect the new release.
+
+### **Announce the release**
+
+Email to the different distribution lists announce@apache.org, user@spot.apache.org, dev@spot.apache.org (using your @apache.org email) For example:
+
+ To: announce@apache.org, user@spot.apache.org, dev@spot.apache.org
+ Subject: [ANNOUNCE] Apache Spot 1.0 (incubating) released
+
+ The Apache Spot (Incubating) team is pleased to announce the release of Spot 1.0-incubating.
+
+ This is the first release of Spot. Major step forward of the project.
+
+ Apache Spot (Incubating) is open source software for leveraging insights from flow and packet analysis. It helps enterprises and service providers gain insight on their network environments through transparency of service delivery and identification of potential security threats or attacks happening among resources operating at cloud scale. While current threat intelligence tools help, identifying unknown threats and attacks remains a challenge.
+
+ The release is available here:
+ http://nolamarketing.com/client/apache-spot/download/ (Update from final version)
+
+ The full change log is available here:
+ https://issues.apache.org/jira/secure/ReleaseNote.jspa?projectId=12320637&version=12340668
+
+ Your help and feedback is more than welcome. For more information on how to report problems and to get involved, visit the project website at http://spot.apache.org/.
+
+ The Apache Spot (Incubating) Team
+
+
+### **Close the Jira Ticket**
+
+Once the release is announced you can go to Jira and Close the EPIC created to perform the release.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/label-bug-enhancement.bmp
----------------------------------------------------------------------
diff --git a/docs/label-bug-enhancement.bmp b/docs/label-bug-enhancement.bmp
deleted file mode 100644
index abff14f..0000000
Binary files a/docs/label-bug-enhancement.bmp and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/label-discussion.bmp
----------------------------------------------------------------------
diff --git a/docs/label-discussion.bmp b/docs/label-discussion.bmp
deleted file mode 100644
index 3ddbc7a..0000000
Binary files a/docs/label-discussion.bmp and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/label-issue-type.bmp
----------------------------------------------------------------------
diff --git a/docs/label-issue-type.bmp b/docs/label-issue-type.bmp
deleted file mode 100644
index 308df82..0000000
Binary files a/docs/label-issue-type.bmp and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/label-repos.bmp
----------------------------------------------------------------------
diff --git a/docs/label-repos.bmp b/docs/label-repos.bmp
deleted file mode 100644
index 3a24847..0000000
Binary files a/docs/label-repos.bmp and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/label-tracking.bmp
----------------------------------------------------------------------
diff --git a/docs/label-tracking.bmp b/docs/label-tracking.bmp
deleted file mode 100644
index ad49559..0000000
Binary files a/docs/label-tracking.bmp and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/labels-design-tracking.bmp
----------------------------------------------------------------------
diff --git a/docs/labels-design-tracking.bmp b/docs/labels-design-tracking.bmp
deleted file mode 100644
index 9b06ac1..0000000
Binary files a/docs/labels-design-tracking.bmp and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/oni-approach.png
----------------------------------------------------------------------
diff --git a/docs/oni-approach.png b/docs/oni-approach.png
deleted file mode 100644
index 0b0fc19..0000000
Binary files a/docs/oni-approach.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/oni-guy.png
----------------------------------------------------------------------
diff --git a/docs/oni-guy.png b/docs/oni-guy.png
deleted file mode 100644
index ae64581..0000000
Binary files a/docs/oni-guy.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/oni-guy_big.png
----------------------------------------------------------------------
diff --git a/docs/oni-guy_big.png b/docs/oni-guy_big.png
deleted file mode 100644
index 9b585dc..0000000
Binary files a/docs/oni-guy_big.png and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/docs/oni-logo.png
----------------------------------------------------------------------
diff --git a/docs/oni-logo.png b/docs/oni-logo.png
deleted file mode 100644
index 23d1e5a..0000000
Binary files a/docs/oni-logo.png and /dev/null differ