You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 21:37:17 UTC
[11/18] incubator-spot git commit: syncing with master
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/proxy.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/proxy.py b/spot-oa/api/resources/proxy.py
new file mode 100644
index 0000000..c162bbf
--- /dev/null
+++ b/spot-oa/api/resources/proxy.py
@@ -0,0 +1,467 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import md5
+import api.resources.impala_engine as ImpalaEngine
+import api.resources.hdfs_client as HDFSClient
+from hdfs.util import HdfsError
+import api.resources.configurator as Configuration
+from collections import defaultdict
+import json
+import os
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connectios related to a request name in one hour
+--------------------------------------------------------------------------
+"""
+def suspicious_requests(date,uri=None,ip=None,limit=250):
+
+ db = Configuration.db()
+ proxy_query = ("""
+ SELECT STRAIGHT_JOIN
+ ps.tdate,ps.time,ps.clientip,ps.host,ps.reqmethod,ps.useragent,
+ ps.resconttype,ps.duration,ps.username,ps.webcat,ps.referer,
+ ps.respcode,ps.uriport,ps.uripath,ps.uriquery,ps.serverip,ps.scbytes,
+ ps.csbytes,ps.fulluri,ps.ml_score,ps.uri_rep,ps.respcode_name,
+ ps.network_context
+ FROM
+ {0}.proxy_scores ps
+ LEFT JOIN
+ {0}.proxy_threat_investigation pt
+ ON (ps.fulluri = pt.fulluri)
+ WHERE
+ ps.y={1} AND ps.m={2} AND ps.d={3}
+ AND (pt.fulluri is NULL)
+ """).format(db,date.year,date.month,date.day)
+
+
+ p_filter = ""
+ p_filter += " AND ps.fulluri LIKE '%{0}%'".format(uri) if uri else ""
+ p_filter += " AND ps.clientip = '{0}'".format(ip) if ip else ""
+ p_filter += " ORDER BY ps.ml_score limit {0}".format(limit)
+ proxy_query = proxy_query + p_filter
+ return ImpalaEngine.execute_query_as_list(proxy_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connectios details for one request.
+--------------------------------------------------------------------------
+"""
+def details(date,uri,ip):
+
+ if not uri and not ip:
+ return None
+
+ db = Configuration.db()
+ p_details = ("""
+ SELECT
+ tdate,time,clientIp,host,webcat,respcode,respcode_name
+ ,reqmethod,useragent,resconttype,referer,uriport,serverip
+ ,scbytes,csbytes,fulluri,hh
+ FROM
+ {0}.proxy_edge
+ WHERE
+ y={1} AND m={2} AND d={3} AND
+ (fulluri='{4}' AND clientIp='{5}')
+ """).format(db,date.year,date.month,date.day,uri.replace("'","//'"),ip)
+ return ImpalaEngine.execute_query_as_list(p_details)
+
+"""
+--------------------------------------------------------------------------
+Score a request
+--------------------------------------------------------------------------
+"""
+def score_request(date,score,uri):
+
+ if not score and not uri:
+ return None
+
+ db = Configuration.db()
+ p_query = ("""
+ SELECT
+ tdate,time,clientip,host,reqmethod,useragent,resconttype
+ ,duration,username,webcat,referer,respcode,uriport
+ ,uripath,uriquery,serverip,scbytes,csbytes,fulluri
+ ,word,ml_score,uri_rep,respcode_name,network_context
+ FROM
+ {0}.proxy_scores
+ WHERE
+ y={1} and m={2} and d={3}
+ AND fulluri = '{4}'
+ """).format(db,date.year,date.month,date.day,uri)
+
+ connections = ImpalaEngine.execute_query(p_query)
+
+ # add score to connections
+ insert_command = ("""
+ INSERT INTO {0}.proxy_threat_investigation PARTITION (y={1},m={2},d={3})
+ VALUES (""") \
+ .format(db,date.year,date.month,date.day)
+
+ fb_data = []
+ first = True
+ num_rows = 0
+ for row in connections:
+ cip_index = row[2]
+ uri_index = row[18]
+ tme_index = row[2]
+ hash_field = [str( md5.new(str(cip_index) + str(uri_index)).hexdigest() \
+ + str((tme_index.split(":"))[0]) )]
+
+ threat_data = (row[0],row[18],score)
+ fb_data.append([row[0],row[1],row[2],row[3],row[4],row[5],row[6],row[7] \
+ ,row[8],row[9],row[10],row[11],row[12],row[13],row[14],row[15] \
+ ,row[16],row[17],row[18],row[19],score,row[20],row[21],row[22], \
+ row[23],hash_field])
+ insert_command += "{0}{1}".format("," if not first else "", threat_data)
+ first = False
+ num_rows += 1
+
+ insert_command += ")"
+ if num_rows > 0: ImpalaEngine.execute_query(insert_command)
+
+ # create feedback file.
+ app_path = Configuration.spot()
+ feedback_path = "{0}/proxy/scored_results/{1}{2}{3}/feedback"\
+ .format(app_path,date.year,str(date.month).zfill(2),str(date.day).zfill(2))
+
+ ap_file = True
+ if len(HDFSClient.list_dir(feedback_path)) == 0:
+ fb_data.insert(0,["p_date","p_time","clientip","host","reqmethod",\
+ "useragent","resconttype","duration","username","webcat","referer",\
+ "respcode","uriport","uripath","uriquery","serverip","scbytes","csbytes",\
+ "fulluri","word","score","uri_rep","uri_sev","respcode_name",\
+ "network_context","hash"])
+ ap_file = False
+
+ HDFSClient.put_file_csv(fb_data,feedback_path,"ml_feedback.csv",append_file=ap_file)
+ return True
+
+"""
+--------------------------------------------------------------------------
+Get expanded search from raw table.
+--------------------------------------------------------------------------
+"""
+def expanded_search(date,uri):
+
+ db = Configuration.db()
+ expanded_query = ("""
+ SELECT p_date, p_time, clientip, username, duration, fulluri,\
+ webcat, respcode, reqmethod,useragent, resconttype,\
+ referer, uriport, serverip, scbytes, csbytes
+ FROM {0}.proxy
+ WHERE y='{1}' AND m='{2}' AND d='{3}'
+ AND (fulluri='{4}' OR referer ='{4}')
+ ORDER BY p_time
+ """)\
+ .format(db,date.year,str(date.month).zfill(2),str(date.day).zfill(2),uri)
+ return ImpalaEngine.execute_query_as_list(expanded_query)
+
+"""
+--------------------------------------------------------------------------
+Get scored request from threat investigation.
+--------------------------------------------------------------------------
+"""
+def get_scored_requests(date):
+
+ db = Configuration.db()
+ sc_query = ("""
+ SELECT
+ tdate,fulluri,uri_sev
+ FROM
+ {0}.proxy_threat_investigation
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ return ImpalaEngine.execute_query_as_list(sc_query)
+
+"""
+--------------------------------------------------------------------------
+Create storyboard.
+Migrated from IPython Notebooks
+--------------------------------------------------------------------------
+"""
+def create_storyboard(uri,date,title,text,expanded_search,top_results):
+
+ clientips = defaultdict(int)
+ reqmethods = defaultdict(int)
+ rescontype = defaultdict(int)
+ referers = defaultdict(int)
+ refered = defaultdict(int)
+ requests = []
+
+
+ for row in expanded_search:
+ clientips[row['clientIp']]+=1
+ reqmethods[row['requestMethod']]+=1
+ rescontype[row['responseContentType']]+=1
+ if row['uri'] == uri:
+ #Source URI's that refered the user to the threat
+ referers[row['referer']]+=1
+ requests += [{'clientip':row['clientIp'], 'referer':row['referer'],'reqmethod':row['requestMethod'], 'resconttype':row['responseContentType']}]
+
+ else:
+ #Destination URI's refered by the threat
+ refered[row['uri']]+=1
+
+ create_incident_progression(uri,requests,refered,date)
+ create_timeline(uri,clientips,date,top_results)
+ save_comments(uri,title,text,date)
+
+ return True
+
+"""
+--------------------------------------------------------------------------
+Create timeline for storyboard
+--------------------------------------------------------------------------
+"""
+def create_timeline(anchor,clientips,date,top_results):
+ response = ""
+ susp_ips = []
+
+ if clientips:
+ srtlist = sorted(list(clientips.items()), key=lambda x: x[1], reverse=True)
+ for val in srtlist[:top_results]:
+ susp_ips.append(val[0])
+
+ if anchor != "":
+ db = Configuration.db()
+ time_line_query = ("""
+ SELECT p_threat,tstart,tend,duration,clientip,respcode,respcodename
+ FROM {0}.proxy_timeline
+ WHERE
+ y={1} AND m={2} AND d={3} AND p_threat != '{4}'
+ """).format(db,date.year,date.month,date.day,anchor.replace("'","//'"))
+
+ tmp_timeline_data = ImpalaEngine.execute_query_as_list(time_line_query)
+
+ imp_query = ("""
+ INSERT INTO TABLE {0}.proxy_timeline
+ PARTITION (y={2}, m={3},d={4})
+ SELECT
+ '{7}' as p_threat, concat(cast(p_date as string),
+ ' ', cast(MIN(p_time) as string)) AS tstart,
+ concat(cast(p_date as string), ' ',
+ cast(MAX(p_time) as string)) AS tend,
+ SUM(duration) AS duration,
+ clientip, respcode,"respCodeName" as respCodeName
+ FROM {0}.proxy
+ WHERE fulluri='{1}' AND clientip IN ({5})
+ AND y='{2}' AND m='{3}' AND d='{4}'
+ GROUP BY clientip, p_time, respcode, p_date
+ LIMIT {6}
+ """)\
+ .format(db,anchor,date.year,str(date.month).zfill(2),\
+ str(date.day).zfill(2),("'" + "','".join(susp_ips) + "'")\
+ ,top_results,anchor)
+
+ app_path = Configuration.spot()
+ old_file = "{0}/proxy/hive/oa/timeline/y={1}/m={2}/d={3}"\
+ .format(app_path,date.year,date.month,date.day)
+
+ HDFSClient.delete_folder(old_file,"impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+
+ #Insert temporary values
+ for item in tmp_timeline_data:
+ insert_query = ("""
+ INSERT INTO {0}.proxy_timeline PARTITION(y={1} , m={2} ,d={3})
+ VALUES ('{4}', '{5}', '{6}',{7},'{8}','{9}','{10}')
+ """)\
+ .format(db,date.year,date.month,date.day,\
+ item["p_threat"],item["tstart"],item["tend"],item["duration"],item["clientip"],item["respcode"],item["respcodename"])
+
+ ImpalaEngine.execute_query(insert_query)
+
+ ImpalaEngine.execute_query(imp_query)
+ response = "Timeline successfully saved"
+ else:
+ response = "Timeline couldn't be created"
+
+"""
+--------------------------------------------------------------------------
+Create inciden progression for storyboard.
+--------------------------------------------------------------------------
+"""
+def create_incident_progression(anchor,requests,referers,date):
+
+ hash_name = md5.new(str(anchor)).hexdigest()
+ file_name = "incident-progression-{0}.json".format(hash_name)
+ app_path = Configuration.spot()
+ hdfs_path = "{0}/proxy/oa/storyboard/{1}/{2}/{3}"\
+ .format(app_path,date.year,date.month,date.day)
+
+ data = {'fulluri':anchor, 'requests':requests,'referer_for':referers.keys()}
+ if HDFSClient.put_file_json(data,hdfs_path,file_name,overwrite_file=True) :
+ response = "Incident progression successfuly created"
+ else:
+ return False
+
+"""
+--------------------------------------------------------------------------
+Save comments for storyboard.
+--------------------------------------------------------------------------
+"""
+def save_comments(uri,title,text,date):
+
+ db = Configuration.db()
+ sb_query = ("""
+ SELECT
+ p_threat,title,text
+ FROM
+ {0}.proxy_storyboard
+ WHERE
+ y = {1} AND m= {2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+ sb_data = ImpalaEngine.execute_query_as_list(sb_query)
+
+
+ # find value if already exists.
+ saved = False
+ for item in sb_data:
+ if item["p_threat"] == uri:
+ item["title"] = title
+ item["text"] = text
+ saved = True
+
+ if not saved:
+ sb_data.append({'text': text, 'p_threat': str(uri), 'title': title})
+
+ #remove old file.
+ app_path = Configuration.spot()
+ old_file = "{0}/proxy/hive/oa/storyboard/y={1}/m={2}/d={3}/"\
+ .format(app_path,date.year,date.month,date.day)
+ HDFSClient.delete_folder(old_file,"impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+
+ for item in sb_data:
+ insert_query = ("""
+ INSERT INTO {0}.proxy_storyboard PARTITION(y={1} , m={2} ,d={3})
+ VALUES ( '{4}', '{5}', '{6}')
+ """)\
+ .format(db,date.year,date.month,date.day,\
+ item["p_threat"],item["title"],item["text"])
+
+ ImpalaEngine.execute_query(insert_query)
+
+"""
+--------------------------------------------------------------------------
+Get storyboard comments.
+--------------------------------------------------------------------------
+"""
+def story_board(date):
+
+ db = Configuration.db()
+ sb_query= ("""
+ SELECT
+ p_threat,title,text
+ FROM
+ {0}.proxy_storyboard
+ WHERE
+ y={1} AND m={2} AND d={3}
+ """).format(db,date.year,date.month,date.day)
+
+ results = ImpalaEngine.execute_query_as_list(sb_query)
+ for row in results:
+ row["text"] = row["text"].replace("\n","\\n")
+ return results
+
+"""
+--------------------------------------------------------------------------
+Get timeline for storyboard.
+--------------------------------------------------------------------------
+"""
+def time_line(date,uri):
+
+ db = Configuration.db()
+ time_line_query = ("""
+ SELECT
+ p_threat,tstart,tend,duration,clientip,respcode,respcodename
+ FROM {0}.proxy_timeline
+ WHERE
+ y={1} AND m={2} AND d={3}
+ AND p_threat = '{4}'
+ """).format(db,date.year,date.month,date.day,uri)
+
+ return ImpalaEngine.execute_query_as_list(time_line_query)
+
+"""
+--------------------------------------------------------------------------
+Get incident progression for storyboard.
+--------------------------------------------------------------------------
+"""
+def incident_progression(date,uri):
+
+ app_path = Configuration.spot()
+ hdfs_path = "{0}/proxy/oa/storyboard/{1}/{2}/{3}".format(app_path,\
+ date.year,date.month,date.day)
+
+ hash_name = md5.new(str(uri)).hexdigest()
+ file_name = "incident-progression-{0}.json".format(hash_name)
+
+ if HDFSClient.file_exists(hdfs_path,file_name):
+ return json.loads(HDFSClient.get_file("{0}/{1}"\
+ .format(hdfs_path,file_name)))
+ else:
+ return {}
+
+"""
+Return a list(dict) with all the data ingested during the time frame provided.
+"""
+def ingest_summary(start_date,end_date):
+
+ db = Configuration.db()
+ is_query = ("""
+ SELECT
+ tdate,total
+ FROM {0}.proxy_ingest_summary
+ WHERE
+ ( y >= {1} and y <= {2}) AND
+ ( m >= {3} and m <= {4}) AND
+ ( d >= {5} and d <= {6})
+ """)\
+ .format(db,start_date.year,end_date.year,start_date.month,end_date.month, start_date.day, end_date.day)
+
+ return ImpalaEngine.execute_query_as_list(is_query)
+
+
+"""
+--------------------------------------------------------------------------
+Reset scored connections.
+--------------------------------------------------------------------------
+"""
+def reset_scored_connections(date):
+
+ proxy_storyboard = "proxy/hive/oa/storyboard"
+ proxy_threat_investigation = "dns_threat_dendro/hive/oa/timeline"
+ proxy_timeline = "proxy/hive/oa/threat_investigation"
+ app_path = Configuration.spot()
+
+ try:
+ # remove parquet files manually to allow the comments update.
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_storyboard,date.year,date.month,date.day) , "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_threat_investigation,date.year,date.month,date.day), "impala")
+ HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+ app_path,proxy_timeline,date.year,date.month,date.day), "impala")
+ ImpalaEngine.execute_query("invalidate metadata")
+ return True
+
+ except HdfsError:
+ return False
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/context/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/context/README.md b/spot-oa/context/README.md
index e4869d2..cf65bdc 100644
--- a/spot-oa/context/README.md
+++ b/spot-oa/context/README.md
@@ -1,6 +1,6 @@
This folder must include the following files:
-###**ipranges.csv**
+### **ipranges.csv**
This is a comma separated file, defines a range of IP that should be considered as part of your internal network.
This file will be requested by the following modules:
@@ -14,7 +14,7 @@ Schema with zero-indexed columns:
1.limit_IP : string
2.name : string
-###**iploc.csv**
+### **iploc.csv**
This is a comma separated geolocation database file containing all (or most) known public IP ranges and the details of its location.
This file is required by the following modules:
@@ -46,7 +46,7 @@ Example: 10.180.88.23
Ip to integer = (10 * 16777216) + (180 * 65536) + (88 * 256) + (23) = 179591191.
-###**networkcontext_1.csv**
+### **networkcontext_1.csv**
This is a comma separated file necessary to add more specific context about your network.
This file will be requested by the following modules:
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/README
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/README b/spot-oa/ipython/README
new file mode 100644
index 0000000..f8f2f7f
--- /dev/null
+++ b/spot-oa/ipython/README
@@ -0,0 +1,10 @@
+This is the IPython directory.
+
+For more information on configuring IPython, do:
+
+ipython -h
+
+or to create an empty default profile, populated with default config files:
+
+ipython profile create
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/extensions/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/extensions/__init__.py b/spot-oa/ipython/extensions/__init__.py
new file mode 100644
index 0000000..ecb1860
--- /dev/null
+++ b/spot-oa/ipython/extensions/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/extensions/spot_webapp.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/extensions/spot_webapp.py b/spot-oa/ipython/extensions/spot_webapp.py
new file mode 100644
index 0000000..2f958cb
--- /dev/null
+++ b/spot-oa/ipython/extensions/spot_webapp.py
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import sys
+from os import path
+
+sys.path.append(path.dirname(path.dirname(path.dirname(__file__))))
+
+from api.graphql.webapp import load_jupyter_server_extension
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/ipython_notebook_config.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/ipython_notebook_config.py b/spot-oa/ipython/profile_spot/ipython_notebook_config.py
new file mode 100644
index 0000000..9f83d97
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/ipython_notebook_config.py
@@ -0,0 +1,570 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import os
+# Configuration file for ipython-notebook.
+
+c = get_config()
+
+#------------------------------------------------------------------------------
+# NotebookApp configuration
+#------------------------------------------------------------------------------
+
+# NotebookApp will inherit config from: BaseIPythonApplication, Application
+
+# The number of additional ports to try if the specified port is not available.
+# c.NotebookApp.port_retries = 50
+
+# Extra variables to supply to jinja templates when rendering.
+# c.NotebookApp.jinja_template_vars = {}
+
+# The url for MathJax.js.
+# c.NotebookApp.mathjax_url = ''
+
+# Supply extra arguments that will be passed to Jinja environment.
+# c.NotebookApp.jinja_environment_options = {}
+
+# The IP address the notebook server will listen on.
+c.NotebookApp.ip = '0.0.0.0'
+
+# DEPRECATED use base_url
+# c.NotebookApp.base_project_url = '/'
+
+# Create a massive crash report when IPython encounters what may be an internal
+# error. The default is to append a short message to the usual traceback
+# c.NotebookApp.verbose_crash = False
+
+# Python modules to load as notebook server extensions. This is an experimental
+# API, and may change in future releases.
+c.NotebookApp.server_extensions = ['extensions.spot_webapp']
+
+# The random bytes used to secure cookies. By default this is a new random
+# number every time you start the Notebook. Set it to a value in a config file
+# to enable logins to persist across server sessions.
+#
+# Note: Cookie secrets should be kept private, do not share config files with
+# cookie_secret stored in plaintext (you can read the value from a file).
+# c.NotebookApp.cookie_secret = ''
+
+# The default URL to redirect to from `/`
+c.NotebookApp.default_url = '/files/ui/flow/suspicious.html'
+
+# Whether to open in a browser after starting. The specific browser used is
+# platform dependent and determined by the python standard library `webbrowser`
+# module, unless it is overridden using the --browser (NotebookApp.browser)
+# configuration option.
+c.NotebookApp.open_browser = False
+
+# The date format used by logging formatters for %(asctime)s
+# c.NotebookApp.log_datefmt = '%Y-%m-%d %H:%M:%S'
+
+# The port the notebook server will listen on.
+c.NotebookApp.port = 8889
+
+# Whether to overwrite existing config files when copying
+# c.NotebookApp.overwrite = False
+
+# The kernel spec manager class to use. Should be a subclass of
+# `IPython.kernel.kernelspec.KernelSpecManager`.
+#
+# The Api of KernelSpecManager is provisional and might change without warning
+# between this version of IPython and the next stable one.
+# c.NotebookApp.kernel_spec_manager_class = <class 'IPython.kernel.kernelspec.KernelSpecManager'>
+
+# Set the Access-Control-Allow-Origin header
+#
+# Use '*' to allow any origin to access your server.
+#
+# Takes precedence over allow_origin_pat.
+# c.NotebookApp.allow_origin = ''
+
+# The notebook manager class to use.
+# c.NotebookApp.contents_manager_class = <class 'IPython.html.services.contents.filemanager.FileContentsManager'>
+
+# Use a regular expression for the Access-Control-Allow-Origin header
+#
+# Requests from an origin matching the expression will get replies with:
+#
+# Access-Control-Allow-Origin: origin
+#
+# where `origin` is the origin of the request.
+#
+# Ignored if allow_origin is set.
+# c.NotebookApp.allow_origin_pat = ''
+
+# The full path to an SSL/TLS certificate file.
+# c.NotebookApp.certfile = u''
+
+# The logout handler class to use.
+# c.NotebookApp.logout_handler_class = <class 'IPython.html.auth.logout.LogoutHandler'>
+
+# The base URL for the notebook server.
+#
+# Leading and trailing slashes can be omitted, and will automatically be added.
+# c.NotebookApp.base_url = '/'
+
+# The session manager class to use.
+# c.NotebookApp.session_manager_class = <class 'IPython.html.services.sessions.sessionmanager.SessionManager'>
+
+# Supply overrides for the tornado.web.Application that the IPython notebook
+# uses.
+c.NotebookApp.tornado_settings = {
+ 'debug': os.environ.get('SPOT_DEV')=='1'
+}
+
+# The directory to use for notebooks and kernels.
+# c.NotebookApp.notebook_dir = u''
+
+# The kernel manager class to use.
+# c.NotebookApp.kernel_manager_class = <class 'IPython.html.services.kernels.kernelmanager.MappingKernelManager'>
+
+# The file where the cookie secret is stored.
+# c.NotebookApp.cookie_secret_file = u''
+
+# Supply SSL options for the tornado HTTPServer. See the tornado docs for
+# details.
+# c.NotebookApp.ssl_options = {}
+
+#
+# c.NotebookApp.file_to_run = ''
+
+# The IPython profile to use.
+# c.NotebookApp.profile = u'default'
+
+# DISABLED: use %pylab or %matplotlib in the notebook to enable matplotlib.
+# c.NotebookApp.pylab = 'disabled'
+
+# Whether to enable MathJax for typesetting math/TeX
+#
+# MathJax is the javascript library IPython uses to render math/LaTeX. It is
+# very large, so you may want to disable it if you have a slow internet
+# connection, or for offline use of the notebook.
+#
+# When disabled, equations etc. will appear as their untransformed TeX source.
+c.NotebookApp.enable_mathjax = False
+
+# Reraise exceptions encountered loading server extensions?
+# c.NotebookApp.reraise_server_extension_failures = False
+
+# The base URL for websockets, if it differs from the HTTP server (hint: it
+# almost certainly doesn't).
+#
+# Should be in the form of an HTTP origin: ws[s]://hostname[:port]
+# c.NotebookApp.websocket_url = ''
+
+# The Logging format template
+# c.NotebookApp.log_format = '[%(name)s]%(highlevel)s %(message)s'
+
+# The name of the IPython directory. This directory is used for logging
+# configuration (through profiles), history storage, etc. The default is usually
+# $HOME/.ipython. This option can also be specified through the environment
+# variable IPYTHONDIR.
+# c.NotebookApp.ipython_dir = u''
+
+# The cluster manager class to use.
+# c.NotebookApp.cluster_manager_class = <class 'IPython.html.services.clusters.clustermanager.ClusterManager'>
+
+# Set the log level by value or name.
+# c.NotebookApp.log_level = 30
+
+# Hashed password to use for web authentication.
+#
+# To generate, type in a python/IPython shell:
+#
+# from IPython.lib import passwd; passwd()
+#
+# The string should be of the form type:salt:hashed-password.
+# c.NotebookApp.password = u''
+
+# extra paths to look for Javascript notebook extensions
+# c.NotebookApp.extra_nbextensions_path = []
+
+# Set the Access-Control-Allow-Credentials: true header
+# c.NotebookApp.allow_credentials = False
+
+# Path to an extra config file to load.
+#
+# If specified, load this config file in addition to any other IPython config.
+# c.NotebookApp.extra_config_file = u''
+
+# Extra paths to search for serving static files.
+#
+# This allows adding javascript/css to be available from the notebook server
+# machine, or overriding individual files in the IPython
+# c.NotebookApp.extra_static_paths = []
+
+# The full path to a private key file for usage with SSL/TLS.
+# c.NotebookApp.keyfile = u''
+
+# Whether to trust or not X-Scheme/X-Forwarded-Proto and X-Real-Ip/X-Forwarded-
+# For headerssent by the upstream reverse proxy. Necessary if the proxy handles
+# SSL
+# c.NotebookApp.trust_xheaders = False
+
+# Extra paths to search for serving jinja templates.
+#
+# Can be used to override templates from IPython.html.templates.
+# c.NotebookApp.extra_template_paths = []
+
+# The config manager class to use
+# c.NotebookApp.config_manager_class = <class 'IPython.html.services.config.manager.ConfigManager'>
+
+# Whether to install the default config files into the profile dir. If a new
+# profile is being created, and IPython contains config files for that profile,
+# then they will be staged into the new directory. Otherwise, default config
+# files will be automatically generated.
+# c.NotebookApp.copy_config_files = False
+
+# The login handler class to use.
+# c.NotebookApp.login_handler_class = <class 'IPython.html.auth.login.LoginHandler'>
+
+# DEPRECATED, use tornado_settings
+# c.NotebookApp.webapp_settings = {}
+
+# Specify what command to use to invoke a web browser when opening the notebook.
+# If not specified, the default browser will be determined by the `webbrowser`
+# standard library module, which allows setting of the BROWSER environment
+# variable to override it.
+# c.NotebookApp.browser = u''
+
+#------------------------------------------------------------------------------
+# KernelManager configuration
+#------------------------------------------------------------------------------
+
+# Manages a single kernel in a subprocess on this host.
+#
+# This version starts kernels with Popen.
+
+# KernelManager will inherit config from: ConnectionFileMixin
+
+# DEPRECATED: Use kernel_name instead.
+#
+# The Popen Command to launch the kernel. Override this if you have a custom
+# kernel. If kernel_cmd is specified in a configuration file, IPython does not
+# pass any arguments to the kernel, because it cannot make any assumptions about
+# the arguments that the kernel understands. In particular, this means that the
+# kernel does not receive the option --debug if it given on the IPython command
+# line.
+# c.KernelManager.kernel_cmd = []
+
+# Should we autorestart the kernel if it dies.
+# c.KernelManager.autorestart = False
+
+# set the stdin (ROUTER) port [default: random]
+# c.KernelManager.stdin_port = 0
+
+# Set the kernel's IP address [default localhost]. If the IP address is
+# something other than localhost, then Consoles on other machines will be able
+# to connect to the Kernel, so be careful!
+# c.KernelManager.ip = u''
+
+# JSON file in which to store connection info [default: kernel-<pid>.json]
+#
+# This file will contain the IP, ports, and authentication key needed to connect
+# clients to this kernel. By default, this file will be created in the security
+# dir of the current profile, but can be specified by absolute path.
+# c.KernelManager.connection_file = ''
+
+# set the control (ROUTER) port [default: random]
+# c.KernelManager.control_port = 0
+
+# set the heartbeat port [default: random]
+# c.KernelManager.hb_port = 0
+
+# set the shell (ROUTER) port [default: random]
+# c.KernelManager.shell_port = 0
+
+#
+# c.KernelManager.transport = 'tcp'
+
+# set the iopub (PUB) port [default: random]
+# c.KernelManager.iopub_port = 0
+
+#------------------------------------------------------------------------------
+# ProfileDir configuration
+#------------------------------------------------------------------------------
+
+# An object to manage the profile directory and its resources.
+#
+# The profile directory is used by all IPython applications, to manage
+# configuration, logging and security.
+#
+# This object knows how to find, create and manage these directories. This
+# should be used by any code that wants to handle profiles.
+
+# Set the profile location directly. This overrides the logic used by the
+# `profile` option.
+# c.ProfileDir.location = u''
+
+#------------------------------------------------------------------------------
+# Session configuration
+#------------------------------------------------------------------------------
+
+# Object for handling serialization and sending of messages.
+#
+# The Session object handles building messages and sending them with ZMQ sockets
+# or ZMQStream objects. Objects can communicate with each other over the
+# network via Session objects, and only need to work with the dict-based IPython
+# message spec. The Session will handle serialization/deserialization, security,
+# and metadata.
+#
+# Sessions support configurable serialization via packer/unpacker traits, and
+# signing with HMAC digests via the key/keyfile traits.
+#
+# Parameters ----------
+#
+# debug : bool
+# whether to trigger extra debugging statements
+# packer/unpacker : str : 'json', 'pickle' or import_string
+# importstrings for methods to serialize message parts. If just
+# 'json' or 'pickle', predefined JSON and pickle packers will be used.
+# Otherwise, the entire importstring must be used.
+#
+# The functions must accept at least valid JSON input, and output *bytes*.
+#
+# For example, to use msgpack:
+# packer = 'msgpack.packb', unpacker='msgpack.unpackb'
+# pack/unpack : callables
+# You can also set the pack/unpack callables for serialization directly.
+# session : bytes
+# the ID of this Session object. The default is to generate a new UUID.
+# username : unicode
+# username added to message headers. The default is to ask the OS.
+# key : bytes
+# The key used to initialize an HMAC signature. If unset, messages
+# will not be signed or checked.
+# keyfile : filepath
+# The file containing a key. If this is set, `key` will be initialized
+# to the contents of the file.
+
+# Username for the Session. Default is your system username.
+# c.Session.username = u'diego'
+
+# The name of the unpacker for unserializing messages. Only used with custom
+# functions for `packer`.
+# c.Session.unpacker = 'json'
+
+# Threshold (in bytes) beyond which a buffer should be sent without copying.
+# c.Session.copy_threshold = 65536
+
+# The name of the packer for serializing messages. Should be one of 'json',
+# 'pickle', or an import name for a custom callable serializer.
+# c.Session.packer = 'json'
+
+# The maximum number of digests to remember.
+#
+# The digest history will be culled when it exceeds this value.
+# c.Session.digest_history_size = 65536
+
+# The UUID identifying this session.
+# c.Session.session = u''
+
+# The digest scheme used to construct the message signatures. Must have the form
+# 'hmac-HASH'.
+# c.Session.signature_scheme = 'hmac-sha256'
+
+# execution key, for signing messages.
+# c.Session.key = ''
+
+# Debug output in the Session
+# c.Session.debug = False
+
+# The maximum number of items for a container to be introspected for custom
+# serialization. Containers larger than this are pickled outright.
+# c.Session.item_threshold = 64
+
+# path to file containing execution key.
+# c.Session.keyfile = ''
+
+# Threshold (in bytes) beyond which an object's buffer should be extracted to
+# avoid pickling.
+# c.Session.buffer_threshold = 1024
+
+# Metadata dictionary, which serves as the default top-level metadata dict for
+# each message.
+# c.Session.metadata = {}
+
+#------------------------------------------------------------------------------
+# MappingKernelManager configuration
+#------------------------------------------------------------------------------
+
+# A KernelManager that handles notebook mapping and HTTP error handling
+
+# MappingKernelManager will inherit config from: MultiKernelManager
+
+# The name of the default kernel to start
+# c.MappingKernelManager.default_kernel_name = 'python2'
+
+#
+# c.MappingKernelManager.root_dir = u''
+
+# The kernel manager class. This is configurable to allow subclassing of the
+# KernelManager for customized behavior.
+# c.MappingKernelManager.kernel_manager_class = 'IPython.kernel.ioloop.IOLoopKernelManager'
+
+#------------------------------------------------------------------------------
+# ContentsManager configuration
+#------------------------------------------------------------------------------
+
+# Base class for serving files and directories.
+#
+# This serves any text or binary file, as well as directories, with special
+# handling for JSON notebook documents.
+#
+# Most APIs take a path argument, which is always an API-style unicode path, and
+# always refers to a directory.
+#
+# - unicode, not url-escaped
+# - '/'-separated
+# - leading and trailing '/' will be stripped
+# - if unspecified, path defaults to '',
+# indicating the root path.
+
+# The base name used when creating untitled files.
+# c.ContentsManager.untitled_file = 'untitled'
+
+# Python callable or importstring thereof
+#
+# To be called on a contents model prior to save.
+#
+# This can be used to process the structure, such as removing notebook outputs
+# or other side effects that should not be saved.
+#
+# It will be called as (all arguments passed by keyword)::
+#
+# hook(path=path, model=model, contents_manager=self)
+#
+# - model: the model to be saved. Includes file contents.
+# Modifying this dict will affect the file that is stored.
+# - path: the API path of the save destination
+# - contents_manager: this ContentsManager instance
+# c.ContentsManager.pre_save_hook = None
+
+#
+# c.ContentsManager.checkpoints_class = <class 'IPython.html.services.contents.checkpoints.Checkpoints'>
+
+# Glob patterns to hide in file and directory listings.
+# c.ContentsManager.hide_globs = [u'__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dylib', '*~']
+
+# The base name used when creating untitled notebooks.
+# c.ContentsManager.untitled_notebook = 'Untitled'
+
+# The base name used when creating untitled directories.
+# c.ContentsManager.untitled_directory = 'Untitled Folder'
+
+#
+# c.ContentsManager.checkpoints = None
+
+#
+# c.ContentsManager.checkpoints_kwargs = {}
+
+#------------------------------------------------------------------------------
+# FileContentsManager configuration
+#------------------------------------------------------------------------------
+
+# FileContentsManager will inherit config from: ContentsManager
+
+#
+# c.FileContentsManager.root_dir = u''
+
+# The base name used when creating untitled files.
+# c.FileContentsManager.untitled_file = 'untitled'
+
+# Python callable or importstring thereof
+#
+# to be called on the path of a file just saved.
+#
+# This can be used to process the file on disk, such as converting the notebook
+# to a script or HTML via nbconvert.
+#
+# It will be called as (all arguments passed by keyword)::
+#
+# hook(os_path=os_path, model=model, contents_manager=instance)
+#
+# - path: the filesystem path to the file just written - model: the model
+# representing the file - contents_manager: this ContentsManager instance
+# c.FileContentsManager.post_save_hook = None
+
+# Python callable or importstring thereof
+#
+# To be called on a contents model prior to save.
+#
+# This can be used to process the structure, such as removing notebook outputs
+# or other side effects that should not be saved.
+#
+# It will be called as (all arguments passed by keyword)::
+#
+# hook(path=path, model=model, contents_manager=self)
+#
+# - model: the model to be saved. Includes file contents.
+# Modifying this dict will affect the file that is stored.
+# - path: the API path of the save destination
+# - contents_manager: this ContentsManager instance
+# c.FileContentsManager.pre_save_hook = None
+
+#
+# c.FileContentsManager.checkpoints_class = <class 'IPython.html.services.contents.checkpoints.Checkpoints'>
+
+# Glob patterns to hide in file and directory listings.
+# c.FileContentsManager.hide_globs = [u'__pycache__', '*.pyc', '*.pyo', '.DS_Store', '*.so', '*.dylib', '*~']
+
+# The base name used when creating untitled notebooks.
+# c.FileContentsManager.untitled_notebook = 'Untitled'
+
+# The base name used when creating untitled directories.
+# c.FileContentsManager.untitled_directory = 'Untitled Folder'
+
+#
+# c.FileContentsManager.checkpoints = None
+
+#
+# c.FileContentsManager.checkpoints_kwargs = {}
+
+# DEPRECATED, use post_save_hook
+# c.FileContentsManager.save_script = False
+
+#------------------------------------------------------------------------------
+# NotebookNotary configuration
+#------------------------------------------------------------------------------
+
+# A class for computing and verifying notebook signatures.
+
+# The number of notebook signatures to cache. When the number of signatures
+# exceeds this value, the oldest 25% of signatures will be culled.
+# c.NotebookNotary.cache_size = 65535
+
+# The secret key with which notebooks are signed.
+# c.NotebookNotary.secret = ''
+
+# The sqlite file in which to store notebook signatures. By default, this will
+# be in your IPython profile. You can set it to ':memory:' to disable sqlite
+# writing to the filesystem.
+# c.NotebookNotary.db_file = u''
+
+# The file where the secret key is stored.
+# c.NotebookNotary.secret_file = u''
+
+# The hashing algorithm used to sign notebooks.
+# c.NotebookNotary.algorithm = 'sha256'
+
+#------------------------------------------------------------------------------
+# KernelSpecManager configuration
+#------------------------------------------------------------------------------
+
+# Whitelist of allowed kernel names.
+#
+# By default, all installed kernels are allowed.
+# c.KernelSpecManager.whitelist = set([])
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/startup/README
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/startup/README b/spot-oa/ipython/profile_spot/startup/README
new file mode 100644
index 0000000..61d4700
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/startup/README
@@ -0,0 +1,11 @@
+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+ 00-first.py
+ 50-middle.py
+ 99-last.ipy
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/startup/graphql.py
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/startup/graphql.py b/spot-oa/ipython/profile_spot/startup/graphql.py
new file mode 100644
index 0000000..9aea6b6
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/startup/graphql.py
@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import IPython
+import json
+import urllib3
+import os
+
+class GraphQLClient:
+ def __init__(self, url=None):
+ self.url = url or 'http://localhost:{}/graphql'.format(self.get_nbserver_info()['port'])
+ self.variables = None
+
+ def get_nbserver_info(self):
+ profile_loc = IPython.config.get_config()['ProfileDir']['location']
+ nbserver_pid = os.getppid()
+ nbserver_file = os.path.join(profile_loc, 'security', 'nbserver-{}.json'.format(nbserver_pid))
+
+ try:
+ return json.load(open(nbserver_file))
+ except:
+ return {}
+
+ def set_query(self, query):
+ self.query = query
+
+ def set_variables(self, variables):
+ self.variables = variables
+
+ def send_query(self):
+ assert(self.url is not None)
+ assert(type(self.url) is str)
+ assert(self.query is not None)
+ assert(type(self.query) is str)
+
+ data = {
+ 'query': self.query
+ }
+
+ if self.variables is not None and type(self.variables) is dict:
+ data['variables'] = self.variables
+
+ encoded_data = json.dumps(data).encode('utf-8')
+
+ http = urllib3.PoolManager()
+
+ response = http.request(
+ 'POST',
+ self.url,
+ body=encoded_data,
+ headers={
+ 'Accept': 'application/json',
+ 'Content-type': 'application/json'
+ }
+ )
+
+ try:
+ return json.loads(response.data.decode('utf-8'))
+ except:
+ return {
+ 'errors': [
+ {
+ 'status': response.status,
+ 'message': 'Failed to contact GraphQL endpoint. Is "{}" the correct URL?'.format(self.url)
+ }
+ ]
+ }
+
+ @classmethod
+ def request(cls, query, variables=None, url=None):
+ client = cls(url)
+
+ client.set_query(query)
+ if variables is not None:
+ client.set_variables(variables)
+
+ return client.send_query()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif b/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif
new file mode 100644
index 0000000..7dc5c7a
Binary files /dev/null and b/spot-oa/ipython/profile_spot/static/custom/ajax-loader.gif differ
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/static/custom/custom.css
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/static/custom/custom.css b/spot-oa/ipython/profile_spot/static/custom/custom.css
new file mode 100755
index 0000000..b8bef4a
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/static/custom/custom.css
@@ -0,0 +1,163 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+
+.spot.spot_easy_mode {
+ background-color: white;
+}
+
+.spot.spot_easy_mode:not(.spot_easy_mode_loading) #site {
+ height: 100% !important;
+}
+
+.spot.spot_easy_mode:not(.spot_easy_mode_loading) #notebook {
+ padding: 0 !important;
+ box-shadow: none;
+ -webkit-box-shadow: none;
+}
+
+.spot.spot_easy_mode #header,
+.spot.spot_easy_mode #pager,
+.spot.spot_easy_mode #tooltip,
+.spot.spot_easy_mode #notebook > .end_space,
+.spot.spot_easy_mode .cell:not(.code_cell),
+.spot.spot_easy_mode .cell.code_cell > .input,
+.spot.spot_easy_mode .cell.code_cell > .widget-area > .prompt {
+ position: absolute !important;
+ top: -1000000000px;
+ left: -1000000000px;
+}
+
+.spot.spot_easy_mode.spot_easy_mode_loading .cell.code_cell {
+ position: absolute !important;
+ top: -1000000000px;
+ left: -1000000000px;
+}
+
+.spot.spot_easy_mode #notebook-container {
+ -webkit-box-shadow: none;
+ box-shadow: none;
+}
+
+.spot.spot_easy_mode .cell.code_cell {
+ border-color: transparent !important;
+ padding: 0 !important;
+}
+
+/*****************************************
+ ** EASY MODE SPINNER **
+ *****************************************/
+.spot #spot_easy_mode_loader {
+ display: none;
+}
+
+.spot.spot_easy_mode #spot_easy_mode_loader {
+ display: block;
+ position: fixed;
+ top: 10px;
+ width: 100%;
+ text-align: center;
+ margin-top: 50px;
+}
+
+.spot.spot_easy_mode #spot_easy_mode_loader_spinner
+{
+ display: inline-block;
+ width: 16px;
+ height: 11px;
+ margin-left: 5px;
+ background-image: url(./ajax-loader.gif);
+}
+
+.spot.spot_easy_mode:not(.spot_easy_mode_loading) #spot_easy_mode_loader
+{
+ right: 25px;
+ margin-top: 0px;
+ width: auto;
+}
+
+.spot.spot_ninja_mode #spot_easy_mode_loader
+{
+ display: none;
+}
+
+/*****************************************
+ ** Spot TOOLTIP STYLING **
+ *****************************************/
+.spot .spot-tooltip .tooltip-inner {
+ word-wrap: break-word;
+}
+
+.spot .spot-text-wrapper.overflown {
+ color: red;
+}
+
+.spot .spot-text-wrapper {
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ width: 12em;
+}
+
+.spot .spot-text-wrapper.spot-text-sm {
+ width: 6em;
+}
+
+.spot .spot-text-wrapper.spot-text-lg {
+ width: 18em;
+}
+
+.spot .spot-text-wrapper.spot-text-xlg {
+ width: 30em;
+}
+
+/* Classes for the threat investigation notebooks*/
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table{
+ margin:10px 10px 20px 0px;
+ font-size:12px;
+}
+
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td, th,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td, th{
+ border: solid 1px #CCC;
+ padding: 3px;
+ background-color: #FFF;
+ line-height:14px;
+}
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table td{
+ height:25px;
+ max-width: 180px;
+ width:100px;
+}
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table th,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area .widget-box div table th{
+
+ background-color: #0071C5;
+ color: #FFF;
+ text-align: center;
+}
+
+body.spot_easy_mode[data-notebook-name*="Threat"] .widget-area > .widget-subarea,
+body.spot_ninja_mode[data-notebook-name*="Threat"] .widget-area > .widget-subarea
+{
+ width: 90%;
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ipython/profile_spot/static/custom/custom.js
----------------------------------------------------------------------
diff --git a/spot-oa/ipython/profile_spot/static/custom/custom.js b/spot-oa/ipython/profile_spot/static/custom/custom.js
new file mode 100755
index 0000000..93579dc
--- /dev/null
+++ b/spot-oa/ipython/profile_spot/static/custom/custom.js
@@ -0,0 +1,351 @@
+// #
+// # Licensed to the Apache Software Foundation (ASF) under one or more
+// # contributor license agreements. See the NOTICE file distributed with
+// # this work for additional information regarding copyright ownership.
+// # The ASF licenses this file to You under the Apache License, Version 2.0
+// # (the "License"); you may not use this file except in compliance with
+// # the License. You may obtain a copy of the License at
+// #
+// # http://www.apache.org/licenses/LICENSE-2.0
+// #
+// # Unless required by applicable law or agreed to in writing, software
+// # distributed under the License is distributed on an "AS IS" BASIS,
+// # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// # See the License for the specific language governing permissions and
+// # limitations under the License.
+// #
+/**
+ * Trigger reload event on parent document
+ */
+function reloadParentData()
+{
+ window.parent.iframeReloadHook && window.parent.iframeReloadHook();
+}
+
+require(['jquery', 'bootstrap'], function($, bootstrap)
+{
+ function retry(query) {
+ var regex = new RegExp(query + '=(\\d)');
+
+ var matcher = regex.exec(window.location.hash);
+
+ var attempt = matcher ? ++matcher[1] : 2;
+
+ // Set hash
+ window.location.hash = '#' + query + '=' + attempt;
+ window.location.reload();
+ }
+
+ // jQuery must be available for easy mode to woark properly
+ if ($===undefined) {
+ if (/#spot_jquery_retry=[015-9]/.test(window.location.hash)) {
+ alert('Notebook was not able to load jQuery after 5 attempts. Please try again later, this is a known issue.');
+ console.warn('Bootstrap\'s tooltip was not loaded.');
+ }
+ else {
+ confirm('Notebook failed to load jQuery, easy mode would not work properly without it. Would you you like to try again?')
+ && retry('spot_jquery_retry');
+ }
+
+ return;
+ }
+
+ if ($.fn.tooltip===undefined) {
+ if (/#spot_bootstrap_retry=\d/.test(window.location.hash)) {
+ alert('Notebook was not able to load bootstrap\'s tooltip plugin. You can still work withuot this feature.');
+ } else {
+ confirm('Notebook was not able to load bootstrap\'s tooltip plugin. You can still work withuot this feature. Would you like to try again?')
+ && retry('spot_bootstrap_retry');
+ }
+ }
+
+ var easyMode = {
+ stage: 0,
+ KERNEL_READY: 1,
+ NOTEBOOK_READY: 2,
+ DOM_READY: 4,
+ ENV_READY: 7,
+ building: false,
+ present: false,
+ ready: false,
+ cells: {
+ total: 0,
+ execution_queue: []
+ }
+ }
+
+ /**
+ * Show EasyMode
+ *
+ * Hide everything but IPython form
+ */
+ function showEasyMode()
+ {
+ $(document.body).addClass('spot_easy_mode').removeClass('spot_ninja_mode');
+ }
+
+ /**
+ * Hide Easy Mode
+ *
+ * Show full IPython Notebook
+ */
+ function hideEasyMode()
+ {
+ $(document.body).addClass('spot_ninja_mode').removeClass('spot_easy_mode');
+ }
+
+ function insertProgressIndicator()
+ {
+ $(document.body).append(
+ '<div id="spot_easy_mode_loader">' +
+ '<span id="spot_easy_mode_loader_details"></span>' +
+ '<span id="spot_easy_mode_loader_spinner"></span>' +
+ '</div>'
+ );
+ }
+
+ /**
+ * Remove progress indicator
+ */
+ function removeProgressIndicator()
+ {
+ $(document.body).removeClass('spot_easy_mode_loading');
+
+ $('#spot_easy_mode_loader').remove();
+ }
+
+ /**
+ * Updates progress indicator's details
+ */
+ function updateProgressIndicator(content)
+ {
+ $('#spot_easy_mode_loader_details').html(content);
+ }
+
+ /**
+ * Show a progress indicator to users
+ */
+ function showBuildingUiIndicator()
+ {
+ $(document.body).addClass('spot_easy_mode_loading');
+
+ insertProgressIndicator();
+
+ updateProgressIndicator(
+ 'Building UI <span id="spot_easy_mode_loader_progress">0</span>%'
+ );
+ }
+
+ /**
+ * Update progress indicator
+ */
+ function updateBuildingUiIndicator()
+ {
+ var p;
+
+ p = (easyMode.cells.total-easyMode.cells.execution_queue.length) * 100 / easyMode.cells.total;
+
+ $('#spot_easy_mode_loader_progress').text(Math.floor(p));
+ }
+
+ function easyModeBootStrap (IPython)
+ {
+ if (easyMode.stage!=easyMode.ENV_READY) return;
+
+ // 1 Build widgets
+ easyMode.building = true;
+
+ console.info('Spot: Building easy mode...');
+
+ // 2 Create an execution queue to display progress
+ easyMode.cells.execution_queue = [];
+
+ easyMode.cells.total = 0;
+ IPython.notebook.get_cells().forEach(function (cell)
+ {
+ if (cell.cell_type==='code')
+ {
+ easyMode.cells.total++;
+ }
+ });
+
+ // Add an extra cell to show progress when requesting execution
+ easyMode.cells.total++;
+
+ // 3 Execute all cells ( Generate UI )
+ IPython.notebook.execute_all_cells();
+
+ updateBuildingUiIndicator();
+ }
+
+ function isEasyModeAvailable()
+ {
+ return window.parent!=window;
+ }
+
+ function isEasyModeEnabled() {
+ return /showEasyMode/.test(window.location.hash);
+ }
+
+ function isNinjaModeEnabled() {
+ return /showNinjaMode/.test(window.location.hash);
+ }
+
+ if (isEasyModeAvailable()) {
+ // Add spot CSS classes (easymode enabled by default)
+ $(document.body).addClass('spot').addClass('spot_easy_mode').addClass('spot_easy_mode_loading');
+
+ // Listen for URL's hash changes
+ $(window).on('hashchange', function ()
+ {
+ if (isEasyModeEnabled()) showEasyMode();
+ else if (isNinjaModeEnabled()) hideEasyMode();
+ });
+
+ $(function () {
+ // Show progress indicator
+ showBuildingUiIndicator();
+ });
+ }
+
+ // Enable spot tooltip text wrapper
+ $(function () {
+ $('body').tooltip({
+ selector: '.spot-text-wrapper[data-toggle]',
+ container: 'body',
+ template: '<div class="spot-tooltip tooltip" role="tooltip"><div class="tooltip-arrow"></div><div class="tooltip-inner"></div></div>',
+ title: function () {
+ return $(this).html();
+ },
+ html: true
+ });
+
+ $('body').on('show.bs.tooltip', '.spot-text-wrapper', function () {
+ return this.clientWidth !== this.scrollWidth || this.clientHeight !== this.scrollHeight;
+ });
+ });
+
+ /**
+ * The following code enables toggle from normal user mode (wizard) and ninja node (notebook UI)
+ */
+ require(['base/js/namespace', 'base/js/events'], function(IPython, events)
+ {
+ // Do nothing when running stand alone
+ if (!isEasyModeAvailable()) return;
+
+ // We are running inside and iframe from Spot. Let's have some fun!
+
+ // Let Notebook be aware it is running on an iframe
+ IPython._target = '_self';
+
+ events.on('kernel_busy.Kernel', function ()
+ {
+ // Skip this event while building UI
+ if (!easyMode.ready) return;
+
+ $('#notebook button.btn:not([disabled])').addClass('spotDisabled').attr('disabled', 'disabled');
+
+ insertProgressIndicator();
+ });
+
+ events.on('kernel_idle.Kernel', function ()
+ {
+ // Skip this event while building UI
+ if (!easyMode.ready) return;
+
+ removeProgressIndicator();
+
+ $('#notebook button.btn.spotDisabled').removeClass('spotDisabled').removeAttr('disabled');
+ });
+
+ events.on('kernel_ready.Kernel', function ()
+ {
+ console.info('Spot: Kernel is ready');
+
+ easyMode.stage |= easyMode.KERNEL_READY;
+
+ easyModeBootStrap(IPython);
+ });
+
+ events.on('notebook_loaded.Notebook', function ()
+ {
+ console.info('Spot: Notebook loaded');
+
+ easyMode.stage |= easyMode.NOTEBOOK_READY;
+
+ easyModeBootStrap(IPython);
+ });
+
+ events.on('shell_reply.Kernel', function (evt, data)
+ {
+ var reply, cell, cellIdx;
+
+ reply = data.reply;
+ cell = easyMode.cells.execution_queue.shift();
+
+ console.log('Last execution status: ' + reply.content.status);
+
+ if ((easyMode.building || easyMode.ready) && reply.content.status==='error')
+ {
+ // First error found
+ easyMode.building = false;
+ easyMode.ready = false;
+ isEasyModeEnabled() && alert('Ooops some code failed. Please go to ipython notebook mode and manually fix the error.');
+ $(document.body).removeClass('spot');
+ removeProgressIndicator();
+ hideEasyMode();
+ // Select and scroll to first cell with errors
+ cellIdx = IPython.notebook.find_cell_index(cell);
+ IPython.notebook.scroll_to_cell(cellIdx);
+ IPython.notebook.select(cellIdx);
+ }
+
+ if (!easyMode.building)
+ {
+ return;
+ }
+
+ if (easyMode.cells.execution_queue.length===0)
+ {
+ console.info('Spot: Cell execution has finished');
+
+ easyMode.ready = true;
+ easyMode.building = false;
+
+ removeProgressIndicator();
+ isEasyModeEnabled() && showEasyMode();
+ }
+ else
+ {
+ updateBuildingUiIndicator();
+ }
+ });
+
+ events.on('execute.CodeCell', function (ev, obj)
+ {
+ var cell;
+
+ if (!easyMode.building && !easyMode.ready) return;
+
+ cell = obj.cell;
+
+ easyMode.cells.execution_queue.push(cell);
+
+ console.info('Spot: Cell execution requested: ' + easyMode.cells.execution_queue.length + ' of ' + easyMode.cells.total);
+
+ cell.clear_output(false);
+ // There seems to be a bug on IPython sometimes cells with widgets dont get cleared
+ // Workaround:
+ cell.output_area.clear_output(false, true);
+ });
+
+ $(function ()
+ {
+ console.info('Spot: DOM is ready');
+
+ easyMode.stage |= easyMode.DOM_READY;
+
+ easyModeBootStrap(IPython);
+ });
+ });
+});
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/INSTALL.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/INSTALL.md b/spot-oa/oa/INSTALL.md
index 342775e..f870a55 100644
--- a/spot-oa/oa/INSTALL.md
+++ b/spot-oa/oa/INSTALL.md
@@ -32,7 +32,7 @@ More information about oa/proxy in [README.md](/spot-oa/oa/proxy)
More information about oa/components in [README.md](/spot-oa/oa/components)
-##Operational Analytics prerequisites
+## Operational Analytics prerequisites
In order to execute this process there are a few prerequisites:
@@ -49,11 +49,11 @@ In order to execute this process there are a few prerequisites:
If users want to implement their own machine learning piece to detect suspicious connections they need to refer
to each data type module to know more about input format and schema.
4. spot-setup project installed. Spot-setup project contains scripts to install hive database and the main configuration
- file for Open Network Insights project.
+ file.
-##Operational Analytics installation and usage
-####Installation
+## Operational Analytics installation and usage
+#### Installation
OA installation consists of the configuration of extra modules or components and creation of a set of files.
Depending on the data type that is going to be processed some components are required and other components are not.
@@ -97,7 +97,7 @@ In order to execute this process there are a few prerequisites:
configure all of them in case new data types are analyzed in the future.
For more details about how to configure each component go to [spot-oa/oa/components/README.md](/spot-oa/oa/components/README.md).
- ####Usage
+ #### Usage
OA process is triggered with the execution of start_oa.py. This Python script will execute the OA process
for only one data type (Flow, DNS or Proxy) at a time. If users need to process multiple data types at the same time, multiple
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/README.md b/spot-oa/oa/components/README.md
index 3353493..ee0e286 100644
--- a/spot-oa/oa/components/README.md
+++ b/spot-oa/oa/components/README.md
@@ -26,7 +26,7 @@ This document will explain the necessary steps to configure the spot-oa componen
| geoloc -> Module to assign geolocation to every IP
-###Data
+### Data
_Data source module._
This module needs to be configured correctly to avoid errors during the spot-oa execution. Here you need to select the correct database engine to obtain the correct results while creating additional details files.
@@ -61,7 +61,7 @@ Example:
}
-###Reputation
+### Reputation
_Reputation check module._
This module is called during spot-oa execution to check the reputation for any given IP, DNS name or URI (depending on the pipeline). The reputation module makes use of two third-party services, McAfee GTI and Facebook ThreatExchange.
@@ -124,7 +124,7 @@ To add a different reputation service, you can read all about it [here](reputati
- app_id: App id to connect to ThreatExchange service.
- app_secret: App secret to connect to ThreatExchange service.
-###IANA
+### IANA
_Internet Assigned Numbers Authority codes translation module._
**Configuration**
@@ -146,7 +146,7 @@ default location, your configuration file should look like this:
}
-###Network Context (nc)
+### Network Context (nc)
_Network Context module._
**Pre-requisites**
@@ -175,7 +175,7 @@ configuration file should look like this:
}
-###Geoloc
+### Geoloc
_Geolocation module._
This is an optional functionality you can enable / disable depending on your preferences.
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/__init__.py b/spot-oa/oa/components/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/__init__.py
+++ b/spot-oa/oa/components/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/data/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/data/__init__.py b/spot-oa/oa/components/data/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/data/__init__.py
+++ b/spot-oa/oa/components/data/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/geoloc/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/geoloc/__init__.py b/spot-oa/oa/components/geoloc/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/geoloc/__init__.py
+++ b/spot-oa/oa/components/geoloc/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/iana/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/iana/__init__.py b/spot-oa/oa/components/iana/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/iana/__init__.py
+++ b/spot-oa/oa/components/iana/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/iana/iana_transform.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/iana/iana_transform.py b/spot-oa/oa/components/iana/iana_transform.py
index 97a1c28..a49c8f9 100644
--- a/spot-oa/oa/components/iana/iana_transform.py
+++ b/spot-oa/oa/components/iana/iana_transform.py
@@ -82,20 +82,20 @@ class IanaTransform(object):
if key in self._qclass_dict:
return self._qclass_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
if column == COL_QTYPE:
if key in self._qtype_dict:
return self._qtype_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
if column == COL_RCODE:
if key in self._rcode_dict:
return self._rcode_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
if column == COL_PRESP:
if key in self._http_rcode_dict:
return self._http_rcode_dict[key]
else:
- return key
+ return "Unknown ({0})".format(key)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/nc/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/nc/__init__.py b/spot-oa/oa/components/nc/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/nc/__init__.py
+++ b/spot-oa/oa/components/nc/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/README.md b/spot-oa/oa/components/reputation/README.md
index 60f3cbd..67cf918 100644
--- a/spot-oa/oa/components/reputation/README.md
+++ b/spot-oa/oa/components/reputation/README.md
@@ -1,4 +1,4 @@
-###Reputation
+### Reputation
This section describes the functionality of the current reputation service modules and how you can implement your own.
It's possible to add new reputation services by implementing a new sub-module, to do that developers should follow
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/__init__.py b/spot-oa/oa/components/reputation/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/reputation/__init__.py
+++ b/spot-oa/oa/components/reputation/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/fb/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/fb/__init__.py b/spot-oa/oa/components/reputation/fb/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/reputation/fb/__init__.py
+++ b/spot-oa/oa/components/reputation/fb/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/fb/fb.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/fb/fb.py b/spot-oa/oa/components/reputation/fb/fb.py
index e9d997c..b33ec7d 100644
--- a/spot-oa/oa/components/reputation/fb/fb.py
+++ b/spot-oa/oa/components/reputation/fb/fb.py
@@ -92,7 +92,7 @@ class Reputation(object):
str_response = urllib2.urlopen(request).read()
response = json.loads(str_response)
except urllib2.HTTPError as e:
- self._logger.error("Error calling ThreatExchange in module fb: " + e.message)
+ self._logger.info("Error calling ThreatExchange in module fb: " + e.message)
reputation_dict[name] = self._get_reputation_label('UNKNOWN')
return reputation_dict
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/gti/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/gti/__init__.py b/spot-oa/oa/components/reputation/gti/__init__.py
index e69de29..ecb1860 100644
--- a/spot-oa/oa/components/reputation/gti/__init__.py
+++ b/spot-oa/oa/components/reputation/gti/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/components/reputation/gti/gti.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/components/reputation/gti/gti.py b/spot-oa/oa/components/reputation/gti/gti.py
index 0e2147f..473ccbd 100644
--- a/spot-oa/oa/components/reputation/gti/gti.py
+++ b/spot-oa/oa/components/reputation/gti/gti.py
@@ -88,6 +88,7 @@ class Reputation(object):
responses += self._call_gti(command, len(queries))
ip_counter = 0
+ category_name_group = ""
for query_resp in responses:
if self.AFLAG_KEY in query_resp or self.REP_KEY not in query_resp :
reputation_dict[values[ip_counter]] = self._get_reputation_label(self.DEFAULT_REP)