You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 21:37:13 UTC
[07/18] incubator-spot git commit: syncing with master
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb b/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
index 5cd89db..8372bbe 100644
--- a/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
+++ b/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
@@ -26,17 +26,10 @@
"except ImportError:\n",
" from IPython.html import widgets\n",
"from IPython.display import display, HTML, clear_output, Javascript \n",
- "\n",
- "with open('/etc/spot.conf') as conf:\n",
- " for line in conf.readlines():\n",
- " if \"DBNAME=\" in line: DBNAME = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
- " elif \"IMPALA_DEM=\" in line: IMPALA_DEM = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
" \n",
"path = os.getcwd().split(\"/\") \n",
"date = path[len(path)-1] \n",
"dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
- "sconnect = dpath + 'proxy_scores.tsv' \n",
- "threat_f = dpath + \"threats.csv\"\n",
"anchor = ''\n",
"anchor_hash = ''\n",
"clientips = defaultdict(int)\n",
@@ -46,7 +39,8 @@
"refered = defaultdict(int)\n",
"requests = []\n",
"top_results = 20\n",
- "details_limit = 1000 "
+ "details_limit = 1000\n",
+ "proxy_comments = {}"
]
},
{
@@ -148,28 +142,48 @@
" clear_output() \n",
" c_uri = []\n",
" uri_sev=[]\n",
- "\n",
- " #discards threats already commented\n",
- " if os.path.isfile(threat_f) and not file_is_empty(threat_f):\n",
- " with open(threat_f, 'r') as th:\n",
- " t_read = csv.reader(th, delimiter='|')\n",
- " t_read.next()\n",
- " for row in t_read: \n",
- " if row[0] != '' : c_uri.append(row[0])\n",
- " \n",
- " with open(sconnect, 'r') as f:\n",
- " reader = csv.reader(f, delimiter='\\t')\n",
- " reader.next()\n",
- " for row in reader:\n",
- " # \"p_date\":0 , \"p_time\":1, \"clientip\":2 , \"host\":3, \"reqmethod\":4 , \"useragent\":5 , \"resconttype\":6\n",
- " # , \"duration\":7, \"username\":8 , \"webcat\":9, \"referer\":10, \"respcode\":11, \"uriport\":12, \"uripath\":13\n",
- " # , \"uriquery\":14, \"serverip\":15, \"scbytes\":16 , \"csbytes\":17, \"fulluri\":18, \"word\":19\n",
- " #Forms a hash out of the anchor to use it as the file name\n",
- " if row[22] == '1': \n",
- " row_hash = md5.new(str(row[18])).hexdigest()\n",
- " if row[18] not in uri_sev and row_hash not in c_uri:\n",
- " uri_sev.append(row[18])\n",
- "\n",
+ " global proxy_comments\n",
+ " \n",
+ " response = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " proxy{\n",
+ " threats{\n",
+ " list(date:$date) {\n",
+ " score\n",
+ " uri\n",
+ " datetime\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " )\n",
+ " proxy_comments = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!) {\n",
+ " proxy{\n",
+ " threats{\n",
+ " comments(date:$date) { \n",
+ " uri\n",
+ " title\n",
+ " text\n",
+ " }\n",
+ " }\n",
+ " }\n",
+ " }\"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ " }\n",
+ " )\n",
+ " proxy_comments = proxy_comments['data']['proxy']['threats']['comments']\n",
+ " if not 'errors' in response: \n",
+ " for row in response['data']['proxy']['threats']['list']: \n",
+ " if row['uri'] not in uri_sev and row['score'] == 1: \n",
+ " uri_sev.append(row['uri'])\n",
+ " else:\n",
+ " print \"An error ocurred: \" + response[\"errors\"][0][\"message\"]\n",
+ " \n",
" threat_title.value =\"<h4>Suspicious URI</h4>\"\n",
" \n",
" if len(uri_sev) == 0:\n",
@@ -191,95 +205,96 @@
" display(topBox) \n",
" \n",
" def search_ip(b): \n",
- " global anchor \n",
- " global anchor_hash\n",
- " global ir_f\n",
+ " global anchor\n",
+ " global expanded_results\n",
" anchor='' \n",
- " anchor_hash = ''\n",
" anchor = susp_select.value \n",
- " anchor_hash = md5.new(str(anchor)).hexdigest()\n",
" removeWidget(3)\n",
" removeWidget(2)\n",
" removeWidget(1) \n",
- " height=80 \n",
- " ir_f = dpath + 'es-' + anchor_hash + \".csv\" \n",
- " table = \"<table><th>TIME</th><th>CLIENT IP</th><th>USERNAME</th><th>DURATION</th> \\\n",
- " <th>FULL URI</th><th>WEB CATEGORY</th><th>RESPONSE CODE</th><th>REQUEST METHOD</th><th>USER AGENT</th> \\\n",
- " <th>MIME TYPE</th><th>REFERER</th><th>URI PORT</th><th>PROXY IP</th><th>SERVER BYTES</th><th>CLIENT BYTES</th>\"\n",
+ " height=80 \n",
" \n",
- " if not os.path.isfile(ir_f) or (os.path.isfile(ir_f) and file_is_empty(ir_f)):\n",
- " # time:0, clientip:1, username:2, duration:3, fullURI:4, webcat:5, respcode:6, reqmethod:7\n",
- " # useragent:8, resconttype: 9, referer: 10, uriport:11, serverip:12, scbytes:13, csbytes:14\n",
- " imp_query = (\"\\\"SELECT p_time, clientip, username, duration, fulluri, webcat, respcode, reqmethod,\\\n",
- " useragent, resconttype, referer, uriport, serverip, scbytes, csbytes FROM {0}.proxy\\\n",
- " WHERE y='{1}' AND m='{2}' AND d='{3}' AND (fulluri='{4}' OR referer ='{4}') ORDER BY p_time\\\"\") \n",
- " \n",
- " imp_query = imp_query.format(DBNAME,yy,mm,dd,anchor) \n",
- " !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
- " !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $ir_f\n",
- " \n",
- " clear_output() \n",
- " req_method = {}\n",
+ " expanded_results = GraphQLClient.request(\n",
+ " query=\"\"\"query($date:SpotDateType!,$uri:String!){\n",
+ " proxy{\n",
+ " threat{\n",
+ " details(date:$date,uri:$uri) {\n",
+ " username\n",
+ " webCategory\n",
+ " responseContentType\n",
+ " datetime\n",
+ " referer\n",
+ " clientToServerBytes\n",
+ " duration\n",
+ " userAgent\n",
+ " uri\n",
+ " serverIp\n",
+ " requestMethod\n",
+ " responseCode\n",
+ " uriPort\n",
+ " clientIp\n",
+ " serverToClientBytes\n",
+ " }\n",
+ " }\n",
+ " } \n",
+ " }\n",
+ " \"\"\",\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'uri': anchor\n",
+ " }\n",
+ " )\n",
+ " \n",
+ " \n",
+ " if not 'errors' in expanded_results: \n",
+ " i = 0\n",
+ " table = \"<table><th>TIME</th><th>CLIENT IP</th><th>USERNAME</th><th>DURATION</th> \\\n",
+ " <th>FULL URI</th><th>WEB CATEGORY</th><th>RESPONSE CODE</th><th>REQUEST METHOD</th><th>USER AGENT</th> \\\n",
+ " <th>MIME TYPE</th><th>REFERER</th><th>URI PORT</th><th>PROXY IP</th><th>SERVER BYTES</th><th>CLIENT BYTES</th>\"\n",
+ " for row in expanded_results['data']['proxy']['threat']['details']:\n",
+ " if i < top_results:\n",
+ " table += \"<tr><td>\"+ str(row['datetime'])+\"</td><td>\"+str(row['clientIp'])+\"</td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['username'])+\"\\\n",
+ " </div></td><td>\"+str(row['duration'])+\"</td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['uri'])+\"</div>\\\n",
+ " </td><td>\"+str(row['webCategory'])+\"</td>\\\n",
+ " <td>\"+str(row['responseCode'])+\"</td><td>\"+str(row['requestMethod'])+\"</td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['userAgent'])+\"</div></td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['responseContentType'])+\"</div></td>\\\n",
+ " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['referer'])+\"</div></td>\\\n",
+ " <td>\"+str(row['uriPort'])+\"</td><td>\"+str(row['serverIp'])+\"</td><td>\\\n",
+ " \"+str(row['serverToClientBytes'])+\"</td><td>\"+str(row['clientToServerBytes'])+\"</td></tr>\"\n",
+ "\n",
+ " height += 20\n",
+ " i+=1\n",
+ " table += \"</table>\" \n",
+ " result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results)\n",
+ " else:\n",
+ " table = \"<table></table>\"\n",
+ " result_html_title.value='<h4>No results were found.</h4>'\n",
"\n",
- " with open(ir_f, 'r') as f:\n",
- " #Creates default dictionaries\n",
- " global reqmethods\n",
- " global rescontype\n",
- " global referers\n",
- " global refered\n",
- " global requests\n",
- " global clientips\n",
- " clientips = defaultdict(int)\n",
- " reqmethods = defaultdict(int)\n",
- " rescontype = defaultdict(int)\n",
- " referers = defaultdict(int)\n",
- " refered = defaultdict(int)\n",
- " try:\n",
- " reader = csv.reader(f, delimiter='\\t')\n",
- " reader.next() # Skip headers\n",
- " i=0 \n",
- " for row in reader:\n",
- " clientips[row[1]]+=1\n",
- " reqmethods[row[7]]+=1\n",
- " rescontype[row[9]]+=1\n",
- " if row[10] != anchor:\n",
- " #Source URI's that refered the user to the threat\n",
- " referers[row[10]]+=1\n",
- " if({'clientip':row[1],'referer':row[10],'reqmethod':row[7],'resconttype':row[9]}) not in requests:\n",
- " requests.append({'clientip':row[1],'referer':row[10],'reqmethod':row[7],'resconttype':row[9]})\n",
- " if i < top_results:\n",
- " table += \"<tr><td>\"+row[0]+\"</td><td>\"+row[1]+\"</td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[2]+\"</div></td><td>\"+row[3]+\"</td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[4]+\"</div></td><td>\"+row[5]+\"</td>\\\n",
- " <td>\"+row[6]+\"</td><td>\"+row[7]+\"</td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[8]+\"</div></td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[9]+\"</div></td>\\\n",
- " <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[10]+\"</div></td>\\\n",
- " <td>\"+row[11]+\"</td><td>\"+row[12]+\"</td><td>\"+row[13]+\"</td><td>\"+row[14]+\"</td></tr>\"\n",
- " else:\n",
- " #Destination URI's refered by the threat\n",
- " refered[row[4]]+=1\n",
- " height += 20\n",
- " i+=1\n",
- " table += \"</table>\" \n",
- " result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results)\n",
- " except:\n",
- " table = \"<table></table>\"\n",
- " result_html_title.value='<h4>No results were found.</h4>'\n",
- " \n",
- " result_html.value=table\n",
- " result_html_box.children = [result_html]\n",
+ " result_html.value=table\n",
+ " result_html_box.children = [result_html]\n",
" \n",
- " display_threat_box(anchor)\n",
- " resultTableBox.children = [result_html_title, result_html_box]\n",
- " display(bottomBox)\n",
+ " display_threat_box(anchor)\n",
+ " resultTableBox.children = [result_html_title, result_html_box]\n",
+ " display(bottomBox)\n",
+ " \n",
+ " \n",
" search_btn.on_click(search_ip)\n",
"\n",
" \n",
- "def display_threat_box(ip): \n",
+ "def display_threat_box(ip): \n",
+ " global expanded_results\n",
+ " global proxy_comments\n",
+ " \n",
+ " title =\"\" \n",
+ " comments = \"\"\n",
+ " title = next((item['title'] for item in proxy_comments if item.get(\"uri\") == ip), \"\")\n",
+ " comments = next((item['text'] for item in proxy_comments if item.get(\"uri\") == ip), \"\")\n",
" result_title.value=\"<h4 class='spot-text-wrapper spot-text-xlg' data-toggle='tooltip'>Threat summary for \" + anchor +\"</h4>\"\n",
- " tc_txt_title = widgets.Text(value='', placeholder='Threat Title', width='100%')\n",
- " tc_txa_summary = widgets.Textarea(value='', height=100, width='95%')\n",
+ " tc_txt_title = widgets.Text(value=title, placeholder='Threat Title', width='100%')\n",
+ " tc_txa_summary = widgets.Textarea(value=comments, height=100, width='95%')\n",
" tc_btn_save = widgets.Button(description='Save', width='65px', layout='width:100%')\n",
" tc_btn_save.button_style = 'primary'\n",
" \n",
@@ -293,75 +308,56 @@
" resultSummaryBox.children = [result_title,result_summary_box]\n",
" \n",
" def save_threat_summary(b):\n",
- " global anchor \n",
- " global anchor_hash \n",
- " if anchor != '': \n",
- " global threat_f\n",
- " if not os.path.exists(threat_f): \n",
- " with open(threat_f, 'w') as comment:\n",
- " comment.write('hash|title|summary\\n')\n",
- " \n",
- " with open(threat_f, 'a') as comment:\n",
- " comment.write(anchor_hash + '|' + tc_txt_title.value + '|' +\n",
- " tc_txa_summary.value.replace('\\n', '\\\\n') + '\\n') \n",
- " \n",
- " display(Javascript(\"$(\\\"option[data-value='\" + anchor +\"']\\\").remove();\")) \n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > .widget-box:gt(0)').remove();\"))\n",
- " \n",
- " response = \"Summary successfully saved\"\n",
- " incident_progression(anchor, anchor_hash)\n",
- " timeline(anchor, anchor_hash)\n",
+ " result_msg = \"\"\n",
+ " threat_title = tc_txt_title.value \n",
+ " threat_comment = tc_txa_summary.value\n",
+ "\n",
+ " \n",
+ " if anchor != '': \n",
+ " mutation=\"\"\"mutation(\n",
+ " $date: SpotDateType, \n",
+ " $uri: String!, \n",
+ " $text: String!, \n",
+ " $title: String!,\n",
+ " $threatDetails: [ProxyThreatDetailsInputType!]!,\n",
+ " $first:Int) \n",
+ " {\n",
+ " proxy {\n",
+ " createStoryboard(input:{\n",
+ " threatDetails: $threatDetails,\n",
+ " date: $date, \n",
+ " uri: $uri, \n",
+ " title: $title, \n",
+ " text: $text,\n",
+ " first:$first})\n",
+ " {success}\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ " variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+ " 'uri': anchor,\n",
+ " 'title': threat_title,\n",
+ " 'text': threat_comment,\n",
+ " 'threatDetails': expanded_results['data']['proxy']['threat']['details'],\n",
+ " 'first':top_results\n",
+ " }\n",
+ "\n",
+ " response = GraphQLClient.request(mutation,variables)\n",
+ " if not 'errors' in response:\n",
+ " start_investigation()\n",
+ " result_msg = \"Story board successfully created\"\n",
+ " else:\n",
+ " result_msg = response['errors'][0]['message'] \n",
" else:\n",
- " response = \"No data selected\" \n",
- " \n",
+ " result_msg = \"No data selected\" \n",
+ "\n",
" susp_select.selected_label = susp_select.options[0]\n",
- " display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
+ " display(widgets.Box((widgets.HTML(value=result_msg, width='100%'),)))\n",
" \n",
" tc_btn_save.on_click(save_threat_summary)\n",
- " \n",
- " \n",
- "def incident_progression(anchor, anchor_hash):\n",
- " file_name = dpath + 'incident-progression-'+anchor_hash+'.json'\n",
- " jsonstring = json.dumps({'fulluri':anchor, 'requests':requests,'referer_for':referers.keys()})\n",
- " if not os.path.exists(file_name):\n",
- " with open(file_name, 'w') as f:\n",
- " f.write(jsonstring) \n",
- " response = \"Incident progression successfuly created\"\n",
- " display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
- " \n",
- " \n",
- "def timeline(anchor, anchor_hash): \n",
- " response = \"\"\n",
- " susp_ips = []\n",
- " if clientips:\n",
- " srtlist = sorted(list(clientips.items()), key=lambda x: x[1], reverse=True)\n",
- " for val in srtlist[:top_results]:\n",
- " susp_ips.append(val[0]) \n",
- " \n",
- " if anchor != \"\":\n",
- " sbdet_f = dpath + \"timeline-\"+anchor_hash+\".tsv\"\n",
- " if not os.path.isfile(sbdet_f) or (os.path.isfile(sbdet_f) and file_is_empty(sbdet_f)): \n",
- " imp_query = \"\\\"SELECT concat(cast(p_date as string), ' ', cast(MIN(p_time) as string)) AS tstart,\\\n",
- " concat(cast(p_date as string), ' ', cast(MAX(p_time) as string)) AS tend, SUM(duration) AS duration,\\\n",
- " clientip, respcode from {0}.proxy WHERE fulluri='{1}' AND clientip IN ({5}) \\\n",
- " AND y='{2}' AND m='{3}' AND d='{4}' GROUP BY clientip, p_time, respcode, p_date ORDER BY clientip\\\n",
- " LIMIT {6}\\\"\"\n",
"\n",
- " imp_query=imp_query.format(DBNAME,anchor,yy,mm,dd,(\"'\" + \"','\".join(susp_ips) + \"'\"), details_limit) \n",
- " !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
- " !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $sbdet_f\n",
- " clear_output()\n",
- " \n",
- " response = \"Timeline successfully saved\"\n",
- " else:\n",
- " response = \"Timeline couldn't be created\"\n",
- " \n",
- " display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
- " data_loader()\n",
" \n",
- "def file_is_empty(path):\n",
- " return os.stat(path).st_size==0\n",
- "\n",
"def removeWidget(index):\n",
" js_command = \"$('.widget-area > .widget-subarea > .widget-box:eq({0})').remove();\".format(index) \n",
" display(Javascript(js_command)) "
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/proxy_conf.json
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_conf.json b/spot-oa/oa/proxy/proxy_conf.json
index 5378939..2228e80 100644
--- a/spot-oa/oa/proxy/proxy_conf.json
+++ b/spot-oa/oa/proxy/proxy_conf.json
@@ -42,14 +42,12 @@
, "serverip":15
, "scbytes":16
, "csbytes":17
- , "fulluri":18
+ , "fulluri":18
, "word":19
- , "score":20
+ , "ml_score":20
, "uri_rep":21
- , "uri_sev":22
- , "respcode_name":23
- , "network_context":24
- , "hash":25
+ , "respcode_name":22
+ , "network_context":23
},
"add_reputation":{
"fulluri":18
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/proxy_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_oa.py b/spot-oa/oa/proxy/proxy_oa.py
index 1324f1a..1fcef19 100644
--- a/spot-oa/oa/proxy/proxy_oa.py
+++ b/spot-oa/oa/proxy/proxy_oa.py
@@ -21,14 +21,18 @@ import json
import shutil
import sys
import datetime
-import csv, math
+import csv, math
from collections import OrderedDict
from utils import Util
from components.data.data import Data
from components.iana.iana_transform import IanaTransform
from components.nc.network_context import NetworkContext
+
+import api.resources.hdfs_client as HDFSClient
+import api.resources.impala_engine as impala
from multiprocessing import Process
import pandas as pd
+from impala.util import as_pandas
import time
import md5
@@ -68,7 +72,6 @@ class OA(object):
# initialize data engine
self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '')
- self._engine = Data(self._db, self._table_name,self._logger)
def start(self):
@@ -78,13 +81,12 @@ class OA(object):
####################
self._create_folder_structure()
+ self._clear_previous_executions()
self._add_ipynb()
self._get_proxy_results()
- self._add_reputation()
- self._add_severity()
+ self._add_reputation()
self._add_iana()
- self._add_network_context()
- self._add_hash()
+ self._add_network_context()
self._create_proxy_scores_csv()
self._get_oa_details()
self._ingest_summary()
@@ -102,12 +104,33 @@ class OA(object):
self._data_path,self._ingest_summary_path,self._ipynb_path = Util.create_oa_folders("proxy",self._date)
+ def _clear_previous_executions(self):
+
+ self._logger.info("Cleaning data from previous executions for the day")
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ table_schema = []
+ HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
+ table_schema=['suspicious', 'edge','threat_investigation', 'timeline', 'storyboard', 'summary' ]
+
+ for path in table_schema:
+ HDFSClient.delete_folder("{0}/{1}/hive/oa/{2}/y={3}/m={4}/d={5}".format(HUSER,self._table_name,path,yr,int(mn),int(dy)),user="impala")
+ impala.execute_query("invalidate metadata")
+
+ #removes Feedback file
+ HDFSClient.delete_folder("{0}/{1}/scored_results/{2}{3}{4}/feedback/ml_feedback.csv".format(HUSER,self._table_name,yr,mn,dy))
+ #removes json files from the storyboard
+ HDFSClient.delete_folder("{0}/{1}/oa/{2}/{3}/{4}/{5}".format(HUSER,self._table_name,"storyboard",yr,mn,dy))
+
+
+
def _add_ipynb(self):
if os.path.isdir(self._ipynb_path):
- self._logger.info("Adding edge investigation IPython Notebook")
- shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+ self._logger.info("Adding advanced mode IPython Notebook")
+ shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
self._logger.info("Adding threat investigation IPython Notebook")
shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
@@ -140,23 +163,23 @@ class OA(object):
self._logger.error("There was an error getting ML results from HDFS")
sys.exit(1)
- # add headers.
- self._logger.info("Adding headers")
- self._proxy_scores_headers = [ str(key) for (key,value) in self._conf['proxy_score_fields'].items() ]
-
self._proxy_scores = self._proxy_results[:]
def _create_proxy_scores_csv(self):
-
- proxy_scores_csv = "{0}/proxy_scores.tsv".format(self._data_path)
- proxy_scores_final = self._proxy_scores[:];
- proxy_scores_final.insert(0,self._proxy_scores_headers)
- Util.create_csv_file(proxy_scores_csv,proxy_scores_final, self._results_delimiter)
-
- # create bk file
- proxy_scores_bu_csv = "{0}/proxy_scores_bu.tsv".format(self._data_path)
- Util.create_csv_file(proxy_scores_bu_csv,proxy_scores_final, self._results_delimiter)
+ # get date parameters.
+ yr = self._date[:4]
+ mn = self._date[4:6]
+ dy = self._date[6:]
+ value_string = ""
+
+ for row in self._proxy_scores:
+ value_string += str(tuple(Util.cast_val(item) for item in row)) + ","
+
+ load_into_impala = ("""
+ INSERT INTO {0}.proxy_scores partition(y={2}, m={3}, d={4}) VALUES {1}
+ """).format(self._db, value_string[:-1], yr, mn, dy)
+ impala.execute_query(load_into_impala)
def _add_reputation(self):
@@ -195,25 +218,22 @@ class OA(object):
for result in rep_services_results:
rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
- self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
+ if rep_results:
+ self._proxy_scores = [ conn + [ rep_results[conn[key]] ] for conn in self._proxy_scores ]
+ else:
+ self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
else:
self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
-
- def _add_severity(self):
- # Add severity column
- self._proxy_scores = [conn + [0] for conn in self._proxy_scores]
-
-
def _add_iana(self):
iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if os.path.isfile(iana_conf_file):
iana_config = json.loads(open(iana_conf_file).read())
proxy_iana = IanaTransform(iana_config["IANA"])
- proxy_rcode_index = self._conf["proxy_score_fields"]["respcode"]
- self._proxy_scores = [ conn + [ proxy_iana.get_name(conn[proxy_rcode_index],"proxy_http_rcode")] for conn in self._proxy_scores ]
+ proxy_rcode_index = self._conf["proxy_score_fields"]["respcode"]
+ self._proxy_scores = [ conn + [proxy_iana.get_name(conn[proxy_rcode_index],"proxy_http_rcode")] for conn in self._proxy_scores ]
else:
self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
@@ -225,96 +245,70 @@ class OA(object):
nc_conf = json.loads(open(nc_conf_file).read())["NC"]
proxy_nc = NetworkContext(nc_conf,self._logger)
ip_dst_index = self._conf["proxy_score_fields"]["clientip"]
- self._proxy_scores = [ conn + [proxy_nc.get_nc(conn[ip_dst_index])] for conn in self._proxy_scores ]
-
+ self._proxy_scores = [ conn + [proxy_nc.get_nc(conn[ip_dst_index])] for conn in self._proxy_scores ]
else:
self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
- def _add_hash(self):
- #A hash string is generated to be used as the file name for the edge files.
- #These fields are used for the hash creation, so this combination of values is treated as
- #a 'unique' connection
- cip_index = self._conf["proxy_score_fields"]["clientip"]
- uri_index = self._conf["proxy_score_fields"]["fulluri"]
- tme_index = self._conf["proxy_score_fields"]["p_time"]
-
- self._proxy_scores = [conn + [str( md5.new(str(conn[cip_index]) + str(conn[uri_index])).hexdigest() + str((conn[tme_index].split(":"))[0]) )] for conn in self._proxy_scores]
-
-
def _get_oa_details(self):
self._logger.info("Getting OA Proxy suspicious details")
# start suspicious connects details process.
p_sp = Process(target=self._get_suspicious_details)
p_sp.start()
-
- # p_sp.join()
+
def _get_suspicious_details(self):
- hash_list = []
+ uri_list = []
iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
if os.path.isfile(iana_conf_file):
iana_config = json.loads(open(iana_conf_file).read())
proxy_iana = IanaTransform(iana_config["IANA"])
for conn in self._proxy_scores:
- conn_hash = conn[self._conf["proxy_score_fields"]["hash"]]
- if conn_hash not in hash_list:
- hash_list.append(conn_hash)
- clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
- fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
- date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
- if len(date) == 3:
- year=date[0]
- month=date[1].zfill(2)
- day=date[2].zfill(2)
- hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
- self._get_proxy_details(fulluri,clientip,conn_hash,year,month,day,hh,proxy_iana)
-
-
- def _get_proxy_details(self,fulluri,clientip,conn_hash,year,month,day,hh,proxy_iana):
-
- limit = 250
- output_delimiter = '\t'
- edge_file ="{0}/edge-{1}-{2}.tsv".format(self._data_path,clientip,conn_hash)
- edge_tmp ="{0}/edge-{1}-{2}.tmp".format(self._data_path,clientip,conn_hash)
-
- if not os.path.isfile(edge_file):
- proxy_qry = ("SELECT p_date, p_time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype, \
- referer, uriport, serverip, scbytes, csbytes, fulluri FROM {0}.{1} WHERE y=\'{2}\' AND m=\'{3}\' AND d=\'{4}\' AND \
- h=\'{5}\' AND fulluri =\'{6}\' AND clientip = \'{7}\' LIMIT {8};").format(self._db,self._table_name, year,month,day,hh,fulluri,clientip,limit)
-
- # execute query
- self._engine.query(proxy_qry,edge_tmp,output_delimiter)
- # add IANA to results.
+ clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
+ fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
+ date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
+ if len(date) == 3:
+ year=date[0]
+ month=date[1].zfill(2)
+ day=date[2].zfill(2)
+ hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
+ self._get_proxy_details(fulluri,clientip,year,month,day,hh,proxy_iana)
+
+
+ def _get_proxy_details(self,fulluri,clientip,year,month,day,hh,proxy_iana):
+ limit = 250
+ value_string = ""
+
+ query_to_load =("""
+ SELECT p_date, p_time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype,
+ referer, uriport, serverip, scbytes, csbytes, fulluri, {5} as hh
+ FROM {0}.{1} WHERE y='{2}' AND m='{3}' AND d='{4}' AND
+ h='{5}' AND fulluri='{6}' AND clientip='{7}' LIMIT {8};
+ """).format(self._db,self._table_name, year,month,day,hh,fulluri.replace("'","\\'"),clientip,limit)
+
+ detail_results = impala.execute_query(query_to_load)
+
+ if proxy_iana:
+ # add IANA to results.
self._logger.info("Adding IANA translation to details results")
- with open(edge_tmp) as proxy_details_csv:
- rows = csv.reader(proxy_details_csv, delimiter=output_delimiter,quotechar='"')
- next(proxy_details_csv)
- update_rows = [[conn[0]] + [conn[1]] + [conn[2]] + [conn[3]] + [conn[4]] + [proxy_iana.get_name(conn[5],"proxy_http_rcode") if proxy_iana else conn[5]] + [conn[6]] + [conn[7]] + [conn[8]] + [conn[9]] + [conn[10]] + [conn[11]] + [conn[12]] + [conn[13]] + [conn[14]] if len(conn) > 0 else [] for conn in rows]
- update_rows = filter(None, update_rows)
- header = ["p_date","p_time","clientip","host","webcat","respcode","reqmethod","useragent","resconttype","referer","uriport","serverip","scbytes","csbytes","fulluri"]
- update_rows.insert(0,header)
-
- # due an issue with the output of the query.
- update_rows = [ [ w.replace('"','') for w in l ] for l in update_rows ]
-
-
- # create edge file.
- self._logger.info("Creating edge file:{0}".format(edge_file))
- with open(edge_file,'wb') as proxy_details_edge:
- writer = csv.writer(proxy_details_edge, quoting=csv.QUOTE_NONE, delimiter=output_delimiter)
- if update_rows:
- writer.writerows(update_rows)
- else:
- shutil.copy(edge_tmp,edge_file)
-
- try:
- os.remove(edge_tmp)
- except OSError:
- pass
+
+ updated_rows = [conn + (proxy_iana.get_name(conn[5],"proxy_http_rcode"),) for conn in detail_results]
+ updated_rows = filter(None, updated_rows)
+ else:
+ updated_rows = [conn + ("") for conn in detail_results ]
+
+ for row in updated_rows:
+ value_string += str(tuple(item for item in row)) + ","
+
+ if value_string != "":
+ query_to_insert=("""
+ INSERT INTO {0}.proxy_edge PARTITION (y={1}, m={2}, d={3}) VALUES ({4});
+ """).format(self._db,year, month, day, value_string[:-1])
+ impala.execute_query(query_to_insert)
+
def _ingest_summary(self):
# get date parameters.
@@ -328,44 +322,36 @@ class OA(object):
result_rows = []
df_filtered = pd.DataFrame()
- ingest_summary_file = "{0}/is_{1}{2}.csv".format(self._ingest_summary_path,yr,mn)
- ingest_summary_tmp = "{0}.tmp".format(ingest_summary_file)
-
- if os.path.isfile(ingest_summary_file):
- df = pd.read_csv(ingest_summary_file, delimiter=',')
- #discards previous rows from the same date
- df_filtered = df[df['date'].str.contains("{0}-{1}-{2}".format(yr, mn, dy)) == False]
- else:
- df = pd.DataFrame()
-
# get ingest summary.
- ingest_summary_qry = ("SELECT p_date, p_time, COUNT(*) as total "
- " FROM {0}.{1}"
- " WHERE y='{2}' AND m='{3}' AND d='{4}' "
- " AND p_date IS NOT NULL AND p_time IS NOT NULL "
- " AND clientip IS NOT NULL AND p_time != '' "
- " AND host IS NOT NULL AND fulluri IS NOT NULL "
- " GROUP BY p_date, p_time;")
-
- ingest_summary_qry = ingest_summary_qry.format(self._db,self._table_name, yr, mn, dy)
- results_file = "{0}/results_{1}.csv".format(self._ingest_summary_path,self._date)
- self._engine.query(ingest_summary_qry,output_file=results_file,delimiter=",")
+
+ query_to_load=("""
+ SELECT p_date, p_time, COUNT(*) as total
+ FROM {0}.{1} WHERE y='{2}' AND m='{3}' AND d='{4}'
+ AND p_date IS NOT NULL AND p_time IS NOT NULL
+ AND clientip IS NOT NULL AND p_time != ''
+ AND host IS NOT NULL AND fulluri IS NOT NULL
+ GROUP BY p_date, p_time;
+ """).format(self._db,self._table_name, yr, mn, dy)
- if os.path.isfile(results_file):
- df_results = pd.read_csv(results_file, delimiter=',')
-
+ results = impala.execute_query(query_to_load)
+
+ if results:
+ df_results = as_pandas(results)
#Forms a new dataframe splitting the minutes from the time column/
df_new = pd.DataFrame([["{0} {1}:{2}".format(val['p_date'], val['p_time'].split(":")[0].zfill(2), val['p_time'].split(":")[1].zfill(2)), int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df_results.iterrows()],columns = ingest_summary_cols)
-
+ value_string = ''
#Groups the data by minute
sf = df_new.groupby(by=['date'])['total'].sum()
df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})
- df_final = df_filtered.append(df_per_min, ignore_index=True)
- df_final.to_csv(ingest_summary_tmp,sep=',', index=False)
-
- os.remove(results_file)
- os.rename(ingest_summary_tmp,ingest_summary_file)
+ df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False)
+ if len(df_final) > 0:
+ query_to_insert=("""
+ INSERT INTO {0}.proxy_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
+ """).format(self._db, yr, mn, dy, tuple(df_final))
+
+ impala.execute_query(query_to_insert)
+
else:
self._logger.info("No data found for the ingest summary")
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/start_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/start_oa.py b/spot-oa/oa/start_oa.py
index b604aab..660814f 100644
--- a/spot-oa/oa/start_oa.py
+++ b/spot-oa/oa/start_oa.py
@@ -20,6 +20,7 @@
import argparse
import os
import sys
+sys.path.append("../")
import logging
from utils import Util
@@ -39,43 +40,43 @@ def main():
start_oa(args)
def start_oa(args):
-
- # setup the main logger for all the OA process.
+
+ # setup the main logger for all the OA process.
logger = Util.get_logger('OA',create_file=False)
- logger.info("-------------------- STARTING OA ---------------------")
- validate_parameters_values(args,logger)
+ logger.info("-------------------- STARTING OA ---------------------")
+ validate_parameters_values(args,logger)
# create data type instance.
module = __import__("{0}.{0}_oa".format(args.type),fromlist=['OA'])
-
- # start OA.
+
+ # start OA.
oa_process = module.OA(args.date,args.limit,logger)
oa_process.start()
-
+
def validate_parameters_values(args,logger):
-
+
logger.info("Validating input parameter values")
#date.
- is_date_ok = True if len(args.date) == 8 else False
+ is_date_ok = True if len(args.date) == 8 else False
# type
dirs = os.walk(script_path).next()[1]
is_type_ok = True if args.type in dirs else False
-
- #limit
+
+ #limit
try:
int(args.limit)
is_limit_ok = True
except ValueError:
is_limit_ok = False
-
- if not is_date_ok: logger.error("date parameter is not correct, please validate it")
+
+ if not is_date_ok: logger.error("date parameter is not correct, please validate it")
if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1)
-
+
if __name__=='__main__':
- main()
\ No newline at end of file
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/utils.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 99b006e..2bed10e 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -121,6 +121,18 @@ class Util(object):
writer.writerows(content)
+ @classmethod
+ def cast_val(self,value):
+ try:
+ val = int(value)
+ except:
+ try:
+ val = float(value)
+ except:
+ val = str(value)
+ return val
+
+
class SecHead(object):
def __init__(self, fp):
self.fp = fp
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/requirements.txt
----------------------------------------------------------------------
diff --git a/spot-oa/requirements.txt b/spot-oa/requirements.txt
index 312360b..9f3afb8 100644
--- a/spot-oa/requirements.txt
+++ b/spot-oa/requirements.txt
@@ -5,7 +5,22 @@ Tld == 0.7.6
# spot-ui dependencies
jsonschema
-ipython == 3.2.0
pyzmq
jinja2
tornado
+mistune
+pygments
+terminado
+ipython == 3.2.1
+
+# GraphQL API dependencies
+flask
+flask-graphql
+graphql-core
+urllib3
+
+# API Resources
+setuptools>=3.4.4
+thrift==0.9.3
+impyla
+hdfs
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/runIpython.sh
----------------------------------------------------------------------
diff --git a/spot-oa/runIpython.sh b/spot-oa/runIpython.sh
index 04a09d9..38a4121 100755
--- a/spot-oa/runIpython.sh
+++ b/spot-oa/runIpython.sh
@@ -1,5 +1,4 @@
-#!/bin/sh
-
+#!/bin/bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
@@ -16,5 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-
-ipython notebook --no-mathjax --profile=ia --port=8889 --ip=0.0.0.0 --no-browser '--NotebookApp.extra_static_paths=["ui/ipython/"]'> ipython.out 2>&1&
+pushd $(dirname $0) > /dev/null
+SPOT_OA_DIR=$(pwd)
+popd > /dev/null
+IPYTHONDIR="$SPOT_OA_DIR/ipython/"
+export IPYTHONDIR
+ipython notebook --profile=spot > ipython.out 2>&1&
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/ui/README.md b/spot-oa/ui/README.md
index 4b35c86..d068b39 100755
--- a/spot-oa/ui/README.md
+++ b/spot-oa/ui/README.md
@@ -68,12 +68,15 @@ Provides tools for interactive visualization, noise filters, white listing, and
+ [Suspicious](#suspicious-4)
+ [Storyboard](#storyboard-4)
+ [Ingest Summary](#ingest-summary-2)
+ + [Notebook](#notebook-1)
* [DNS Stores](#dns-stores)
+ [Suspicious](#suspicious-5)
+ [Storyboard](#storyboard-5)
+ + [Notebook](#notebook-2)
* [Proxy Stores](#proxy-stores)
+ [Suspicious](#suspicious-6)
+ [Storyboard](#storyboard-6)
+ + [Notebook](#notebook-3)
## Intended Audience
@@ -91,7 +94,7 @@ Here you will find useful information to get you started on how to contribute to
## Technical Documentation
-Our UI is build on top of Flux application architecture principles, having an understanding of this pattern is highly recommended, for more information about Flux, please go to [Flux web page](https://facebook.github.io/flux/docs/overview.html).
+Our UI is built on top of Flux application architecture principles, having an understanding of this pattern is highly recommended, for more information about Flux, please go to [Flux web page](https://facebook.github.io/flux/docs/overview.html).
### ReactJS + Flux
@@ -104,12 +107,13 @@ For more information about ReactJS and Flux, please go to:
From now on we assume you are familiar with ReactJS+Flux applications.
-For every path found on this document, "SPOT" refers to the path where Spot UI is intalled and "PIPELINE" the pipeline subfolder you want to work on.
+For every path found on this document, "SPOT" refers to the path where Spot UI is installed and "PIPELINE" the pipeline subfolder you want to work on.
#### Development/Debugging process
1. Install Spot UI. Follow this [guide](INSTALL.md#install-spot-ui).
2. Run Spot UI. Follow this [guide](INSTALL.md#how-to-run-spot-ui).
+ 1. Set SPOT_DEV env var to 1 to enable GraphiQL UI and run ipython in debug mode.
3. Start watching for code changes
1. $ cd SPOT/ui/PIPELINE/
2. Watch one of the following modules
@@ -1034,6 +1038,13 @@ _SPOT/ui/js/flow/stores/_
Extends [RestStore.js](#reststore.js)
+###### Notebook
+
+spot-oa/ui/flow/js/stores/NotebookStore.js
+
+spot-oa/ui/flow/js/components/ScoreNotebook.react.js
+
+
##### DNS Stores
_SPOT/ui/js/dns/stores/_
@@ -1095,6 +1106,12 @@ _SPOT/ui/js/dns/stores/_
> - SELECT_COMMENT
> - RELOAD_COMMENTS
+###### Notebook
+
+spot-oa/ui/dns/js/stores/NotebookStore.js
+
+spot-oa/ui/dns/js/components/ScoreNotebook.react.js
+
##### Proxy Stores
_SPOT/ui/js/proxy/_
@@ -1153,3 +1170,9 @@ _SPOT/ui/js/proxy/_
> - UPDATE_DATE
> - SELECT_COMMENT
> - RELOAD_COMMENTS
+
+###### Notebook
+
+spot-oa/ui/proxy/js/stores/NotebookStore.js
+
+spot-oa/ui/proxy/js/components/ScoreNotebook.react.js
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/bootstrap-spot.min.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/bootstrap-spot.min.css b/spot-oa/ui/css/bootstrap-spot.min.css
index 6d05096..cbd2297 100644
--- a/spot-oa/ui/css/bootstrap-spot.min.css
+++ b/spot-oa/ui/css/bootstrap-spot.min.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
/*! Generated by Live LESS Theme Customizer */
.label,sub,sup{vertical-align:baseline}
body,figure{margin:0}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/dendrogram.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/dendrogram.css b/spot-oa/ui/css/dendrogram.css
index 06782df..0395c08 100644
--- a/spot-oa/ui/css/dendrogram.css
+++ b/spot-oa/ui/css/dendrogram.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
.dendrogram {
overflow-y: auto;
overflow-x: auto;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/main.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/main.css b/spot-oa/ui/css/main.css
index 00368ca..8421f13 100755
--- a/spot-oa/ui/css/main.css
+++ b/spot-oa/ui/css/main.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
/* All devices */
.spot-maximized {
width: 100% !important;
@@ -47,6 +63,7 @@
.spot-frame > .spot-frame-content > .panel > .panel-body > .panel-body-container {
width: 100%;
overflow-x: auto;
+ height: 1px;
}
.spot-frame .panel-body-container.container-box > * {
@@ -57,6 +74,39 @@
border: none;
}
+.sortable {
+ display: flex;
+ flex-flow: column;
+ height: 100%;
+}
+
+.overflowing {
+ overflow-y: scroll;
+}
+
+#accordion {
+ height: 85%;
+}
+
+.fit {
+ height: 50%;
+}
+
+::-webkit-scrollbar {
+ width: 7px;
+ height: 7px;
+}
+
+::-webkit-scrollbar-track {
+ -webkit-box-shadow: inset 0 0 6px rgba(0,0,0,0.3);
+ border-radius: 10px;
+}
+
+::-webkit-scrollbar-thumb {
+ border-radius: 10px;
+ -webkit-box-shadow: inset 0 0 10px rgba(0,0,0,0.5);
+}
+
/* Extra small devices (phones, less than 768px) */
/* Small devices (tablets, 768px and up) */
@@ -291,3 +341,24 @@ prevent the active panel from collapsing
.spot-text-wrapper.spot-text-lg {
width: 18em;
}
+
+/* GLYPHICON INPUT */
+
+.inner-addon {
+ position: relative;
+}
+
+.inner-addon .glyphicon {
+ position: absolute;
+ padding: 10px;
+ pointer-events: none;
+}
+
+.left-addon .glyphicon { left: 0px;}
+.right-addon .glyphicon { right: 0px;}
+
+.right-addon input { padding-right: 30px; }
+
+.margin-up-down {
+ padding: 20px 0px;
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/ipython_notebook.html b/spot-oa/ui/dns/ipython_notebook.html
new file mode 100644
index 0000000..21e4572
--- /dev/null
+++ b/spot-oa/ui/dns/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>DNS :: Advanced mode</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Font awesome ICONS-->
+ <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+
+ <style>
+ .spot-row {
+ height: 100%;
+ }
+
+ td.srcIP_rep, td.dstIP_rep {
+ white-space: nowrap;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: DNS :: Advanced mode</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+ <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js b/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
index 462e040..ad40124 100755
--- a/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
@@ -1,5 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ContentLoaderMixin = require('../../../js/components/ContentLoaderMixin.react');
@@ -29,7 +43,7 @@ const DetailsDendrogramPanel = React.createClass({
state.data = {
id: 'root',
- name: DendrogramStore.getSrcIp(),
+ name: DendrogramStore.getClientIp(),
children: []
};
@@ -45,7 +59,7 @@ const DetailsDendrogramPanel = React.createClass({
};
state.data.children.push(childNode);
- answers = item.dns_a.split('|');
+ answers = item.dns_a;
if (answers.length) {
let childId = 0;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsPanel.react.js b/spot-oa/ui/dns/js/components/DetailsPanel.react.js
index 550c516..15fc621 100755
--- a/spot-oa/ui/dns/js/components/DetailsPanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js b/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
index 59c5e23..0ed697e 100755
--- a/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
@@ -30,7 +45,6 @@ var DetailsTablePanel = React.createClass({
},
_render_dns_a_cell: function (answers)
{
- answers = (answers || "").split('|');
answers = answers.map(function (answer, idx)
{
return (
@@ -42,9 +56,6 @@ var DetailsTablePanel = React.createClass({
return answers;
},
- _render_dns_qry_class_cell: false,
- _render_dns_qry_rcode_cell: false,
- _render_dns_qry_type_cell: false,
_onChange: function ()
{
this.replaceState(DetailsStore.getData());
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/FilterInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/FilterInput.react.js b/spot-oa/ui/dns/js/components/FilterInput.react.js
index 666fc3e..3599603 100755
--- a/spot-oa/ui/dns/js/components/FilterInput.react.js
+++ b/spot-oa/ui/dns/js/components/FilterInput.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js b/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
index da17da7..ca322cc 100755
--- a/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
+++ b/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
@@ -8,8 +23,8 @@ const DendrogramMixin = require('../../../js/components/DendrogramMixin.react');
const IncidentProgressionStore = require('../stores/IncidentProgressionStore');
const fieldMapper = {
- ip_dst: 'dns_qry_name',
- dns_qry_name: 'ip_dst'
+ clientIp: 'dnsQuery',
+ dnsQuery: 'clientIp'
};
const IncidentProgressionPanel = React.createClass({
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js b/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
index 7328c76..d2ab8bd 100755
--- a/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
+++ b/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
@@ -46,7 +61,7 @@ function getNodesFromData(data) {
nodes[id] = {
id: id,
label: item[field],
- internalIp: field==='tld',
+ internalIp: field==='tld' ? 1 : 0,
hits: 1
};
}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/ScoreNotebook.react.js b/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
new file mode 100644
index 0000000..e6c5259
--- /dev/null
+++ b/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
@@ -0,0 +1,206 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var SuspiciousStore = require('../stores/SuspiciousStore');
+var NotebookStore = require('../stores/NotebookStore');
+var EdInActions = require('../../../js/actions/EdInActions');
+var SelectInput = require('../../../js/components/scoring/SelectInput.react');
+var FilterSelectInput = require('../../../js/components/scoring/FilterSelectInput.react');
+var ButtonsScoring = require('../../../js/components/scoring/ButtonsScoring.react');
+var RatingInput = require('../../../js/components/scoring/RatingInput.react');
+var SearchGlobalInput = require('../../../js/components/scoring/SearchGlobalInput.react');
+var ScoreMessage = require('../../../js/components/scoring/ScoreMessage.react');
+
+var SpotUtils = require('../../../js/utils/SpotUtils');
+
+var ScoreNotebook = React.createClass({
+ // mixins: [GridPanelMixin],
+ emptySetMessage: 'There is no data available for selected date.',
+ propTypes: {
+ date: React.PropTypes.string.isRequired,
+ },
+ getInitialState: function () {
+ return {
+ scoredEmelents: []
+ };
+ },
+ componentDidMount: function() {
+ SuspiciousStore.addChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ this.setState({size: NotebookStore.completeClass})
+ },
+ componentWillUnmount: function () {
+ SuspiciousStore.removeChangeDataListener(this._onChange);
+ NotebookStore.addChangeDataListener(this._onChange);
+ },
+ render: function () {
+ var content, state, data, cssCls, dnsNameArr = [], dstIpArr = [];
+ state = this.state || {};
+
+ if (state.error)
+ {
+ content = (
+ <div className="text-center text-danger">
+ {state.error}
+ </div>
+ );
+ }
+ else if (state.loading)
+ {
+ content = (
+ <div className="spot-loader">
+ Loading <span className="spinner"></span>
+ </div>
+ );
+ }
+ else if (!state.data || state.data.length === 0)
+ {
+ content = (
+ <div className="text-center">
+ {this.emptySetMessage || ''}
+ </div>
+ );
+ }
+ else
+ {
+ state.data.map((obj) => {
+ if(dstIpArr.indexOf(obj.ip_dst) === -1) {
+ dstIpArr.push(obj.ip_dst);
+ }
+ if(dnsNameArr.indexOf(obj.dns_qry_name) === -1) {
+ dnsNameArr.push(obj.dns_qry_name);
+ }
+ });
+
+ data = [
+ {value: 1, name: 'High', radioName: 'optradio', selected: true},
+ {value: 2, name: 'Medium', radioName: 'optradio', selected: false},
+ {value: 3, name: 'Low', radioName: 'optradio', selected: false}
+ ];
+
+ content = (
+ <div>
+ <div className="margin-up-down">
+ <SearchGlobalInput col="6" maxlength="255"/>
+ <RatingInput data={data} col="6"/>
+ </div>
+ <div className="margin-up-down">
+ <ButtonsScoring name="Score" action="score" onChange={this.score.bind(this)} col="3"/>
+ <ButtonsScoring name="Save" action="save" onChange={this.save.bind(this)} col="3"/>
+ <ButtonsScoring name="Reset Scoring" action="reset" onChange={this.reset.bind(this)} col="3"/>
+ </div>
+ <div className="margin-up-down">
+ <FilterSelectInput nameBox="Client Ip" idSelect="#dstIp" idInput="dstIpIn" col="3"/>
+ <FilterSelectInput nameBox="Query" idSelect="#query" idInput="queryIn" col="9"/>
+ </div>
+ <div className="margin-up-down">
+ <SelectInput title="Client IP" who="dstIp" options={dstIpArr} col="3"/>
+ <SelectInput title="Query" who="query" options={dnsNameArr} col="9"/>
+ </div>
+ <div className="margin-up-down">
+ <ScoreMessage who="scoreMsg"/>
+ </div>
+ </div>
+ );
+ }
+ cssCls = this.state.size ? 'col-md-6 col-lg-6 col-xs-12' : 'col-md-offset-3 col-lg-offset-3 col-md-6 col-lg-6 col-xs-12';
+
+ return(
+ <div className={cssCls + ' spot-frame'}>
+ {content}
+ </div>
+ )
+ },
+ _onChange: function() {
+ const data = SuspiciousStore.getData();
+ this.setState(data);
+ },
+ reset: function() {
+ swal({
+ title: 'Are you sure?',
+ text: "You won't be able to revert this!",
+ type: 'warning',
+ showCancelButton: true,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Yes, reset all!'
+ }).then(() => {
+ EdInActions.resetScoring(SpotUtils.getCurrentDate());
+ swal({
+ title: 'Done!',
+ text: "All scores have been reset.",
+ type: 'success',
+ showCancelButton: false,
+ confirmButtonColor: '#3085d6',
+ cancelButtonColor: '#d33',
+ confirmButtonText: 'Ok!'
+ }).then(() => {
+ this.setState({loading: true});
+ });
+ });
+ },
+ save: function() {
+ let variables = [];
+
+ if (this.state.scoredEmelents.length === 0) {
+ swal('Warning.','You need to score at least 1 threat.','warning');
+ } else {
+ this.state.scoredEmelents.map((row) => {
+ variables.push({
+ 'date': SpotUtils.getCurrentDate(),
+ 'clientIp': row[0] || null,
+ 'dnsQuery': row[1] || null,
+ 'score': row[2]
+ });
+ });
+
+ EdInActions.saveScoring(variables);
+ $('#scoreMsg').addClass("hidden");
+ this.setState({scoredEmelents: [], loading: true});
+ }
+ },
+ score: function() {
+ //this should be changed to take all data at the time, time needed.
+ let dataScored = this.state.scoredEmelents || [];
+ let quickIpScoring = document.getElementById('globalTxt').value;
+ let query = document.getElementById('query').value;
+ let dstIp = document.getElementById('dstIp').value;
+ let rating = $('input[name="optradio"]:checked').val();
+
+ //callback from the father
+ if(quickIpScoring !== '') {
+ dataScored.push([dstIp, quickIpScoring, rating]);
+ } else {
+ dataScored.push([dstIp, query, rating]);
+ }
+
+ this.removeSelected([dstIp, quickIpScoring,query]);
+ $('#scoreMsg').removeClass("hidden");
+ this.setState({scoredEmelents: dataScored});
+ },
+ removeSelected: function(data) {
+ //when an user score, all elements selected need to be removed.
+ data.map((element) => data.map((e) => e !== '' ? $(`option[value="${e}"]`).remove() : ''));
+ $(".select-picker, #globalTxt").val('');
+ }
+
+});
+
+
+module.exports = ScoreNotebook;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js b/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
index 951319e..a2d106f 100755
--- a/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
+++ b/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
var React = require('react');
@@ -26,28 +41,6 @@ var SuspiciousPanel = React.createClass({
</p>
);
},
- /**
- * Answers cell can have multiple pipe separated values. To allow
- * proper displaying of information lets place each answer inside
- * a block element let browsers decide how to display them.
- */
- _render_dns_a_cell: function (answers, item, idx)
- {
- var cellBody;
-
- answers = (answers || []).split('|');
-
- cellBody = answers.map(function (answer, i)
- {
- return (
- <div key={'answer_' + idx + '_' + i}>
- {answer}
- </div>
- );
- }.bind(this));
-
- return cellBody;
- },
_render_ip_dst_cell: function (ip_dst, item, idx)
{
var ip_dst_info, iconClass;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/constants/DnsConstants.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/constants/DnsConstants.js b/spot-oa/ui/dns/js/constants/DnsConstants.js
deleted file mode 100755
index 9e269fa..0000000
--- a/spot-oa/ui/dns/js/constants/DnsConstants.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var DnsConstants = {
- // API URLS
- API_SUSPICIOUS: '../../data/dns/${date}/dns_scores.csv',
- API_DETAILS: '../../data/dns/${date}/edge-${dns_qry_name}_${time}.csv',
- API_VISUAL_DETAILS: '../../data/dns/${date}/dendro-${ip_dst}.csv',
- API_COMMENTS: '../../data/dns/${date}/threats.csv',
- API_INCIDENT_PROGRESSION: '../../data/dns/${date}/threat-dendro-${id}.csv'
-};
-
-module.exports = DnsConstants;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/notebooks.js b/spot-oa/ui/dns/js/notebooks.js
new file mode 100644
index 0000000..81d923e
--- /dev/null
+++ b/spot-oa/ui/dns/js/notebooks.js
@@ -0,0 +1,72 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+ (
+ <form className="form-inline">
+ <div className="form-group">
+ <label htmlFor="dataDatePicker">Data Date:</label>
+ <div className="input-group input-group-xs">
+ <DateInput id="dataDatePicker" />
+ <div className="input-group-addon">
+ <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+ </div>
+ </div>
+ </div>
+ </form>
+ ),
+ document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+ <div id="spot-content">
+ <PanelRow maximized>
+ <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+ <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="dns/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+ </Panel>
+ </PanelRow>
+ </div>,
+ document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/CommentsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/CommentsStore.js b/spot-oa/ui/dns/js/stores/CommentsStore.js
index 291af68..47e2eea 100755
--- a/spot-oa/ui/dns/js/stores/CommentsStore.js
+++ b/spot-oa/ui/dns/js/stores/CommentsStore.js
@@ -1,33 +1,73 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-var d3 = require('d3');
-
-var DnsConstants = require('../constants/DnsConstants');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var RestStore = require('../../../js/stores/RestStore');
-
-var CommentsStore = assign(new RestStore(DnsConstants.API_COMMENTS), {
- _parser: d3.dsv('|', 'text/plain'),
- errorMessages: {
- 404: 'Please choose a different date, no comments have been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(DnsConstants.API_COMMENTS.replace('${date}', date.replace(/-/g, '')));
- }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+
+class CommentsStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType) {
+ dns {
+ threats{
+ comments(date:$date) {
+ title
+ summary: text
+ ...QueryComment
+ ...ClientIpComment
+ }
+ }
+ }
+ }
+
+ fragment QueryComment on DnsQueryCommentType {
+ dnsQuery
+ }
+
+ fragment ClientIpComment on DnsClientIpCommentType {
+ clientIp
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.dns.threats.comments;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+}
+
+const cs = new CommentsStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- CommentsStore.setDate(action.date);
+ cs.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- CommentsStore.reload();
+ cs.sendQuery();
break;
}
});
-module.exports = CommentsStore;
+module.exports = cs;