You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 21:37:13 UTC

[07/18] incubator-spot git commit: syncing with master

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb b/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
index 5cd89db..8372bbe 100644
--- a/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
+++ b/spot-oa/oa/proxy/ipynb_templates/Threat_Investigation_master.ipynb
@@ -26,17 +26,10 @@
     "except ImportError:\n",
     "    from IPython.html import widgets\n",
     "from IPython.display import display, HTML, clear_output, Javascript \n",
-    "\n",
-    "with open('/etc/spot.conf') as conf:\n",
-    "    for line in conf.readlines():\n",
-    "        if \"DBNAME=\" in line: DBNAME = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\");      \n",
-    "        elif \"IMPALA_DEM=\" in line: IMPALA_DEM = line.split(\"=\")[1].strip('\\n').replace(\"'\",\"\"); \n",
     "            \n",
     "path = os.getcwd().split(\"/\") \n",
     "date = path[len(path)-1]   \n",
     "dpath = '/'.join(['data' if var == 'ipynb' else var for var in path]) + '/'\n",
-    "sconnect = dpath + 'proxy_scores.tsv' \n",
-    "threat_f = dpath + \"threats.csv\"\n",
     "anchor = ''\n",
     "anchor_hash = ''\n",
     "clientips  = defaultdict(int)\n",
@@ -46,7 +39,8 @@
     "refered    = defaultdict(int)\n",
     "requests = []\n",
     "top_results = 20\n",
-    "details_limit = 1000  "
+    "details_limit = 1000\n",
+    "proxy_comments = {}"
    ]
   },
   {
@@ -148,28 +142,48 @@
     "    clear_output() \n",
     "    c_uri = []\n",
     "    uri_sev=[]\n",
-    "\n",
-    "    #discards threats already commented\n",
-    "    if os.path.isfile(threat_f) and not file_is_empty(threat_f):\n",
-    "        with open(threat_f, 'r') as th:\n",
-    "            t_read = csv.reader(th, delimiter='|')\n",
-    "            t_read.next()\n",
-    "            for row in t_read: \n",
-    "                if row[0] != '' : c_uri.append(row[0])\n",
-    "            \n",
-    "    with open(sconnect, 'r') as f:\n",
-    "        reader = csv.reader(f, delimiter='\\t')\n",
-    "        reader.next()\n",
-    "        for row in reader:\n",
-    "        #   \"p_date\":0 , \"p_time\":1, \"clientip\":2 , \"host\":3, \"reqmethod\":4 , \"useragent\":5 , \"resconttype\":6\n",
-    "        # , \"duration\":7, \"username\":8 , \"webcat\":9, \"referer\":10, \"respcode\":11, \"uriport\":12, \"uripath\":13\n",
-    "        # , \"uriquery\":14, \"serverip\":15, \"scbytes\":16 , \"csbytes\":17, \"fulluri\":18, \"word\":19\n",
-    "            #Forms a hash out of the anchor to use it as the file name\n",
-    "            if row[22] == '1': \n",
-    "                row_hash = md5.new(str(row[18])).hexdigest()\n",
-    "                if row[18] not in uri_sev and row_hash not in c_uri:\n",
-    "                    uri_sev.append(row[18])\n",
-    "\n",
+    "    global proxy_comments\n",
+    "    \n",
+    "    response = GraphQLClient.request(\n",
+    "        query=\"\"\"query($date:SpotDateType!) {\n",
+    "                 proxy{\n",
+    "                    threats{\n",
+    "                        list(date:$date) {\n",
+    "                            score\n",
+    "                            uri\n",
+    "                            datetime\n",
+    "                        }\n",
+    "                    }\n",
+    "            }\n",
+    "        }\"\"\",\n",
+    "        variables={\n",
+    "            'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+    "        }\n",
+    "    )\n",
+    "    proxy_comments = GraphQLClient.request(\n",
+    "        query=\"\"\"query($date:SpotDateType!) {\n",
+    "                 proxy{\n",
+    "                    threats{\n",
+    "                        comments(date:$date) {                            \n",
+    "                            uri\n",
+    "                            title\n",
+    "                            text\n",
+    "                        }\n",
+    "                    }\n",
+    "            }\n",
+    "        }\"\"\",\n",
+    "        variables={\n",
+    "            'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+    "        }\n",
+    "    )\n",
+    "    proxy_comments = proxy_comments['data']['proxy']['threats']['comments']\n",
+    "    if not 'errors' in response: \n",
+    "        for row in response['data']['proxy']['threats']['list']:        \n",
+    "            if row['uri'] not in uri_sev and row['score'] == 1: \n",
+    "                uri_sev.append(row['uri'])\n",
+    "    else:\n",
+    "        print \"An error ocurred: \" + response[\"errors\"][0][\"message\"]\n",
+    " \n",
     "    threat_title.value =\"<h4>Suspicious URI</h4>\"\n",
     "                       \n",
     "    if len(uri_sev) == 0:\n",
@@ -191,95 +205,96 @@
     "    display(topBox) \n",
     "  \n",
     "    def search_ip(b):  \n",
-    "        global anchor  \n",
-    "        global anchor_hash\n",
-    "        global ir_f\n",
+    "        global anchor\n",
+    "        global expanded_results\n",
     "        anchor='' \n",
-    "        anchor_hash = ''\n",
     "        anchor = susp_select.value   \n",
-    "        anchor_hash = md5.new(str(anchor)).hexdigest()\n",
     "        removeWidget(3)\n",
     "        removeWidget(2)\n",
     "        removeWidget(1) \n",
-    "        height=80        \n",
-    "        ir_f = dpath + 'es-' + anchor_hash + \".csv\" \n",
-    "        table = \"<table><th>TIME</th><th>CLIENT IP</th><th>USERNAME</th><th>DURATION</th> \\\n",
-    "        <th>FULL URI</th><th>WEB CATEGORY</th><th>RESPONSE CODE</th><th>REQUEST METHOD</th><th>USER AGENT</th> \\\n",
-    "        <th>MIME TYPE</th><th>REFERER</th><th>URI PORT</th><th>PROXY IP</th><th>SERVER BYTES</th><th>CLIENT BYTES</th>\"\n",
+    "        height=80   \n",
     "        \n",
-    "        if not os.path.isfile(ir_f) or (os.path.isfile(ir_f) and file_is_empty(ir_f)):\n",
-    "            # time:0, clientip:1, username:2, duration:3, fullURI:4, webcat:5, respcode:6, reqmethod:7\n",
-    "            # useragent:8, resconttype: 9, referer: 10, uriport:11, serverip:12, scbytes:13, csbytes:14\n",
-    "            imp_query = (\"\\\"SELECT p_time, clientip, username, duration, fulluri, webcat, respcode, reqmethod,\\\n",
-    "                 useragent, resconttype, referer, uriport, serverip, scbytes, csbytes FROM {0}.proxy\\\n",
-    "                 WHERE y='{1}' AND m='{2}' AND d='{3}' AND (fulluri='{4}' OR referer ='{4}') ORDER BY p_time\\\"\")   \n",
-    "            \n",
-    "            imp_query = imp_query.format(DBNAME,yy,mm,dd,anchor) \n",
-    "            !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
-    "            !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $ir_f\n",
-    "           \n",
-    "        clear_output() \n",
-    "        req_method = {}\n",
+    "        expanded_results = GraphQLClient.request(\n",
+    "            query=\"\"\"query($date:SpotDateType!,$uri:String!){\n",
+    "                              proxy{\n",
+    "                                threat{\n",
+    "                                  details(date:$date,uri:$uri) {\n",
+    "                                    username\n",
+    "                                    webCategory\n",
+    "                                    responseContentType\n",
+    "                                    datetime\n",
+    "                                    referer\n",
+    "                                    clientToServerBytes\n",
+    "                                    duration\n",
+    "                                    userAgent\n",
+    "                                    uri\n",
+    "                                    serverIp\n",
+    "                                    requestMethod\n",
+    "                                    responseCode\n",
+    "                                    uriPort\n",
+    "                                    clientIp\n",
+    "                                    serverToClientBytes\n",
+    "                                  }\n",
+    "                                }\n",
+    "                              }  \n",
+    "                            }\n",
+    "                            \"\"\",\n",
+    "                        variables={\n",
+    "                        'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+    "                        'uri': anchor\n",
+    "                        }\n",
+    "                    )\n",
+    "        \n",
+    "        \n",
+    "        if not 'errors' in expanded_results: \n",
+    "            i = 0\n",
+    "            table = \"<table><th>TIME</th><th>CLIENT IP</th><th>USERNAME</th><th>DURATION</th> \\\n",
+    "            <th>FULL URI</th><th>WEB CATEGORY</th><th>RESPONSE CODE</th><th>REQUEST METHOD</th><th>USER AGENT</th> \\\n",
+    "            <th>MIME TYPE</th><th>REFERER</th><th>URI PORT</th><th>PROXY IP</th><th>SERVER BYTES</th><th>CLIENT BYTES</th>\"\n",
+    "            for row in expanded_results['data']['proxy']['threat']['details']:\n",
+    "                if i < top_results:\n",
+    "                    table += \"<tr><td>\"+ str(row['datetime'])+\"</td><td>\"+str(row['clientIp'])+\"</td>\\\n",
+    "                    <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['username'])+\"\\\n",
+    "                    </div></td><td>\"+str(row['duration'])+\"</td>\\\n",
+    "                    <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['uri'])+\"</div>\\\n",
+    "                    </td><td>\"+str(row['webCategory'])+\"</td>\\\n",
+    "                    <td>\"+str(row['responseCode'])+\"</td><td>\"+str(row['requestMethod'])+\"</td>\\\n",
+    "                    <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['userAgent'])+\"</div></td>\\\n",
+    "                    <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['responseContentType'])+\"</div></td>\\\n",
+    "                    <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+str(row['referer'])+\"</div></td>\\\n",
+    "                    <td>\"+str(row['uriPort'])+\"</td><td>\"+str(row['serverIp'])+\"</td><td>\\\n",
+    "                    \"+str(row['serverToClientBytes'])+\"</td><td>\"+str(row['clientToServerBytes'])+\"</td></tr>\"\n",
+    "\n",
+    "                height += 20\n",
+    "                i+=1\n",
+    "            table += \"</table>\"                \n",
+    "            result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results)\n",
+    "        else:\n",
+    "            table = \"<table></table>\"\n",
+    "            result_html_title.value='<h4>No results were found.</h4>'\n",
     "\n",
-    "        with open(ir_f, 'r') as f:\n",
-    "            #Creates default dictionaries\n",
-    "            global reqmethods\n",
-    "            global rescontype\n",
-    "            global referers\n",
-    "            global refered\n",
-    "            global requests\n",
-    "            global clientips\n",
-    "            clientips  = defaultdict(int)\n",
-    "            reqmethods = defaultdict(int)\n",
-    "            rescontype = defaultdict(int)\n",
-    "            referers   = defaultdict(int)\n",
-    "            refered    = defaultdict(int)\n",
-    "            try:\n",
-    "                reader = csv.reader(f, delimiter='\\t')\n",
-    "                reader.next() # Skip headers\n",
-    "                i=0         \n",
-    "                for row in reader:\n",
-    "                    clientips[row[1]]+=1\n",
-    "                    reqmethods[row[7]]+=1\n",
-    "                    rescontype[row[9]]+=1\n",
-    "                    if row[10] != anchor:\n",
-    "                        #Source URI's that refered the user to the threat\n",
-    "                        referers[row[10]]+=1\n",
-    "                        if({'clientip':row[1],'referer':row[10],'reqmethod':row[7],'resconttype':row[9]}) not in requests:\n",
-    "                            requests.append({'clientip':row[1],'referer':row[10],'reqmethod':row[7],'resconttype':row[9]})\n",
-    "                        if i < top_results:\n",
-    "                            table += \"<tr><td>\"+row[0]+\"</td><td>\"+row[1]+\"</td>\\\n",
-    "                            <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[2]+\"</div></td><td>\"+row[3]+\"</td>\\\n",
-    "                            <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[4]+\"</div></td><td>\"+row[5]+\"</td>\\\n",
-    "                            <td>\"+row[6]+\"</td><td>\"+row[7]+\"</td>\\\n",
-    "                            <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[8]+\"</div></td>\\\n",
-    "                            <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[9]+\"</div></td>\\\n",
-    "                            <td><div class='spot-text-wrapper' data-toggle='tooltip'>\"+row[10]+\"</div></td>\\\n",
-    "                            <td>\"+row[11]+\"</td><td>\"+row[12]+\"</td><td>\"+row[13]+\"</td><td>\"+row[14]+\"</td></tr>\"\n",
-    "                    else:\n",
-    "                        #Destination URI's refered by the threat\n",
-    "                        refered[row[4]]+=1\n",
-    "                    height += 20\n",
-    "                    i+=1\n",
-    "                table += \"</table>\"                \n",
-    "                result_html_title.value='<h4>Displaying top {0} search results</h4>'.format(top_results)\n",
-    "            except:\n",
-    "                table = \"<table></table>\"\n",
-    "                result_html_title.value='<h4>No results were found.</h4>'\n",
-    "                \n",
-    "            result_html.value=table\n",
-    "            result_html_box.children = [result_html]\n",
+    "        result_html.value=table\n",
+    "        result_html_box.children = [result_html]\n",
     " \n",
-    "            display_threat_box(anchor)\n",
-    "            resultTableBox.children = [result_html_title, result_html_box]\n",
-    "            display(bottomBox)\n",
+    "        display_threat_box(anchor)\n",
+    "        resultTableBox.children = [result_html_title, result_html_box]\n",
+    "        display(bottomBox)\n",
+    "        \n",
+    "        \n",
     "    search_btn.on_click(search_ip)\n",
     "\n",
     "        \n",
-    "def display_threat_box(ip):    \n",
+    "def display_threat_box(ip):   \n",
+    "    global expanded_results\n",
+    "    global proxy_comments\n",
+    "    \n",
+    "    title =\"\" \n",
+    "    comments = \"\"\n",
+    "    title = next((item['title'] for item in proxy_comments if item.get(\"uri\") == ip), \"\")\n",
+    "    comments = next((item['text'] for item in proxy_comments if item.get(\"uri\") == ip), \"\")\n",
     "    result_title.value=\"<h4 class='spot-text-wrapper spot-text-xlg' data-toggle='tooltip'>Threat summary for \" + anchor +\"</h4>\"\n",
-    "    tc_txt_title = widgets.Text(value='', placeholder='Threat Title', width='100%')\n",
-    "    tc_txa_summary = widgets.Textarea(value='', height=100, width='95%')\n",
+    "    tc_txt_title = widgets.Text(value=title, placeholder='Threat Title', width='100%')\n",
+    "    tc_txa_summary = widgets.Textarea(value=comments, height=100, width='95%')\n",
     "    tc_btn_save = widgets.Button(description='Save', width='65px', layout='width:100%')\n",
     "    tc_btn_save.button_style = 'primary'\n",
     "    \n",
@@ -293,75 +308,56 @@
     "    resultSummaryBox.children = [result_title,result_summary_box]\n",
     "    \n",
     "    def save_threat_summary(b):\n",
-    "        global anchor \n",
-    "        global anchor_hash \n",
-    "        if anchor != '':      \n",
-    "            global threat_f\n",
-    "            if not os.path.exists(threat_f):  \n",
-    "                with open(threat_f, 'w') as comment:\n",
-    "                    comment.write('hash|title|summary\\n')\n",
-    "            \n",
-    "            with open(threat_f, 'a') as comment:\n",
-    "                comment.write(anchor_hash + '|' + tc_txt_title.value + '|' +\n",
-    "                                  tc_txa_summary.value.replace('\\n', '\\\\n') + '\\n') \n",
-    "            \n",
-    "            display(Javascript(\"$(\\\"option[data-value='\" + anchor +\"']\\\").remove();\"))   \n",
-    "            display(Javascript(\"$('.widget-area > .widget-subarea > .widget-box:gt(0)').remove();\"))\n",
-    "            \n",
-    "            response = \"Summary successfully saved\"\n",
-    "            incident_progression(anchor, anchor_hash)\n",
-    "            timeline(anchor, anchor_hash)\n",
+    "        result_msg = \"\"\n",
+    "        threat_title = tc_txt_title.value \n",
+    "        threat_comment = tc_txa_summary.value\n",
+    "\n",
+    "        \n",
+    "        if anchor != '':   \n",
+    "            mutation=\"\"\"mutation(\n",
+    "                        $date: SpotDateType, \n",
+    "                        $uri: String!, \n",
+    "                        $text: String!, \n",
+    "                        $title: String!,\n",
+    "                        $threatDetails: [ProxyThreatDetailsInputType!]!,\n",
+    "                        $first:Int) \n",
+    "                        {\n",
+    "                          proxy {\n",
+    "                            createStoryboard(input:{\n",
+    "                                threatDetails: $threatDetails,\n",
+    "                                date: $date, \n",
+    "                                uri: $uri, \n",
+    "                                title: $title, \n",
+    "                                text: $text,\n",
+    "                                first:$first})\n",
+    "                            {success}\n",
+    "                          }\n",
+    "                        }\"\"\"\n",
+    "\n",
+    "            variables={\n",
+    "                'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
+    "                'uri': anchor,\n",
+    "                'title': threat_title,\n",
+    "                'text': threat_comment,\n",
+    "                'threatDetails': expanded_results['data']['proxy']['threat']['details'],\n",
+    "                'first':top_results\n",
+    "                }\n",
+    "\n",
+    "            response = GraphQLClient.request(mutation,variables)\n",
+    "            if not 'errors' in response:\n",
+    "                start_investigation()\n",
+    "                result_msg = \"Story board successfully created\"\n",
+    "            else:\n",
+    "                result_msg = response['errors'][0]['message'] \n",
     "        else:\n",
-    "            response = \"No data selected\" \n",
-    "       \n",
+    "            result_msg = \"No data selected\" \n",
+    "\n",
     "        susp_select.selected_label = susp_select.options[0]\n",
-    "        display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
+    "        display(widgets.Box((widgets.HTML(value=result_msg, width='100%'),)))\n",
     "    \n",
     "    tc_btn_save.on_click(save_threat_summary)\n",
-    "    \n",
-    "    \n",
-    "def incident_progression(anchor, anchor_hash):\n",
-    "    file_name = dpath + 'incident-progression-'+anchor_hash+'.json'\n",
-    "    jsonstring = json.dumps({'fulluri':anchor, 'requests':requests,'referer_for':referers.keys()})\n",
-    "    if not os.path.exists(file_name):\n",
-    "         with open(file_name, 'w') as f:\n",
-    "            f.write(jsonstring)        \n",
-    "    response = \"Incident progression successfuly created\"\n",
-    "    display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
-    "    \n",
-    "    \n",
-    "def timeline(anchor, anchor_hash): \n",
-    "    response = \"\"\n",
-    "    susp_ips = []\n",
-    "    if clientips:\n",
-    "        srtlist = sorted(list(clientips.items()), key=lambda x: x[1], reverse=True)\n",
-    "        for val in srtlist[:top_results]:\n",
-    "            susp_ips.append(val[0])  \n",
-    "            \n",
-    "    if anchor != \"\":\n",
-    "        sbdet_f = dpath + \"timeline-\"+anchor_hash+\".tsv\"\n",
-    "        if not os.path.isfile(sbdet_f) or (os.path.isfile(sbdet_f) and file_is_empty(sbdet_f)):  \n",
-    "            imp_query = \"\\\"SELECT concat(cast(p_date as string), ' ', cast(MIN(p_time) as string)) AS tstart,\\\n",
-    "            concat(cast(p_date as string), ' ', cast(MAX(p_time) as string)) AS tend, SUM(duration) AS duration,\\\n",
-    "            clientip, respcode from {0}.proxy WHERE fulluri='{1}' AND clientip IN ({5}) \\\n",
-    "            AND y='{2}' AND m='{3}' AND d='{4}' GROUP BY clientip, p_time, respcode, p_date ORDER BY clientip\\\n",
-    "            LIMIT {6}\\\"\"\n",
     "\n",
-    "            imp_query=imp_query.format(DBNAME,anchor,yy,mm,dd,(\"'\" + \"','\".join(susp_ips) + \"'\"), details_limit)  \n",
-    "            !impala-shell -i $IMPALA_DEM --quiet -q \"INVALIDATE METADATA\"\n",
-    "            !impala-shell -i $IMPALA_DEM --quiet --print_header -B --output_delimiter='\\t' -q $imp_query -o $sbdet_f\n",
-    "            clear_output()\n",
-    "            \n",
-    "            response = \"Timeline successfully saved\"\n",
-    "    else:\n",
-    "        response = \"Timeline couldn't be created\"\n",
-    "    \n",
-    "    display(widgets.Box((widgets.HTML(value=response, width='100%'),)))\n",
-    "    data_loader()\n",
     "    \n",
-    "def file_is_empty(path):\n",
-    "    return os.stat(path).st_size==0\n",
-    "\n",
     "def removeWidget(index):\n",
     "    js_command = \"$('.widget-area > .widget-subarea > .widget-box:eq({0})').remove();\".format(index)    \n",
     "    display(Javascript(js_command)) "

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/proxy_conf.json
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_conf.json b/spot-oa/oa/proxy/proxy_conf.json
index 5378939..2228e80 100644
--- a/spot-oa/oa/proxy/proxy_conf.json
+++ b/spot-oa/oa/proxy/proxy_conf.json
@@ -42,14 +42,12 @@
         , "serverip":15
         , "scbytes":16
         , "csbytes":17
-        , "fulluri":18
+        , "fulluri":18 
         , "word":19
-        , "score":20
+        , "ml_score":20
         , "uri_rep":21
-        , "uri_sev":22
-        , "respcode_name":23
-        , "network_context":24
-        , "hash":25
+        , "respcode_name":22
+        , "network_context":23 
     },
     "add_reputation":{
         "fulluri":18   

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/proxy/proxy_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_oa.py b/spot-oa/oa/proxy/proxy_oa.py
index 1324f1a..1fcef19 100644
--- a/spot-oa/oa/proxy/proxy_oa.py
+++ b/spot-oa/oa/proxy/proxy_oa.py
@@ -21,14 +21,18 @@ import json
 import shutil
 import sys
 import datetime
-import csv, math
+import csv, math 
 from collections import OrderedDict
 from utils import Util
 from components.data.data import Data
 from components.iana.iana_transform import IanaTransform
 from components.nc.network_context import NetworkContext
+
+import api.resources.hdfs_client as HDFSClient
+import api.resources.impala_engine as impala
 from multiprocessing import Process
 import pandas as pd 
+from impala.util import as_pandas
 
 import time
 import md5
@@ -68,7 +72,6 @@ class OA(object):
 
         # initialize data engine
         self._db = self._spot_conf.get('conf', 'DBNAME').replace("'", "").replace('"', '')
-        self._engine = Data(self._db, self._table_name,self._logger)
 
 
     def start(self):
@@ -78,13 +81,12 @@ class OA(object):
         ####################
 
         self._create_folder_structure()
+        self._clear_previous_executions()   
         self._add_ipynb()
         self._get_proxy_results()
-        self._add_reputation()
-        self._add_severity()
+        self._add_reputation() 
         self._add_iana()
-        self._add_network_context()
-        self._add_hash()
+        self._add_network_context() 
         self._create_proxy_scores_csv()
         self._get_oa_details()
         self._ingest_summary()
@@ -102,12 +104,33 @@ class OA(object):
         self._data_path,self._ingest_summary_path,self._ipynb_path = Util.create_oa_folders("proxy",self._date)
 
 
+    def _clear_previous_executions(self):
+        
+        self._logger.info("Cleaning data from previous executions for the day")       
+        yr = self._date[:4]
+        mn = self._date[4:6]
+        dy = self._date[6:]  
+        table_schema = []
+        HUSER = self._spot_conf.get('conf', 'HUSER').replace("'", "").replace('"', '')
+        table_schema=['suspicious', 'edge','threat_investigation', 'timeline', 'storyboard', 'summary' ] 
+
+        for path in table_schema:
+            HDFSClient.delete_folder("{0}/{1}/hive/oa/{2}/y={3}/m={4}/d={5}".format(HUSER,self._table_name,path,yr,int(mn),int(dy)),user="impala")        
+        impala.execute_query("invalidate metadata")
+
+        #removes Feedback file
+        HDFSClient.delete_folder("{0}/{1}/scored_results/{2}{3}{4}/feedback/ml_feedback.csv".format(HUSER,self._table_name,yr,mn,dy))
+        #removes json files from the storyboard
+        HDFSClient.delete_folder("{0}/{1}/oa/{2}/{3}/{4}/{5}".format(HUSER,self._table_name,"storyboard",yr,mn,dy))
+
+
+
     def _add_ipynb(self):
 
         if os.path.isdir(self._ipynb_path):
 
-            self._logger.info("Adding edge investigation IPython Notebook")
-            shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+            self._logger.info("Adding advanced mode IPython Notebook")
+            shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
 
             self._logger.info("Adding threat investigation IPython Notebook")
             shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
@@ -140,23 +163,23 @@ class OA(object):
             self._logger.error("There was an error getting ML results from HDFS")
             sys.exit(1)
 
-        # add headers.
-        self._logger.info("Adding headers")
-        self._proxy_scores_headers = [  str(key) for (key,value) in self._conf['proxy_score_fields'].items() ]
-
         self._proxy_scores = self._proxy_results[:]
 
 
     def _create_proxy_scores_csv(self):
-
-        proxy_scores_csv = "{0}/proxy_scores.tsv".format(self._data_path)
-        proxy_scores_final = self._proxy_scores[:];
-        proxy_scores_final.insert(0,self._proxy_scores_headers)
-        Util.create_csv_file(proxy_scores_csv,proxy_scores_final, self._results_delimiter)
-
-        # create bk file
-        proxy_scores_bu_csv = "{0}/proxy_scores_bu.tsv".format(self._data_path)
-        Util.create_csv_file(proxy_scores_bu_csv,proxy_scores_final, self._results_delimiter)
+        # get date parameters.
+        yr = self._date[:4]
+        mn = self._date[4:6]
+        dy = self._date[6:] 
+        value_string = ""
+ 
+        for row in self._proxy_scores:
+            value_string += str(tuple(Util.cast_val(item) for item in row)) + ","              
+    
+        load_into_impala = ("""
+             INSERT INTO {0}.proxy_scores partition(y={2}, m={3}, d={4}) VALUES {1}
+        """).format(self._db, value_string[:-1], yr, mn, dy) 
+        impala.execute_query(load_into_impala)
 
 
     def _add_reputation(self):
@@ -195,25 +218,22 @@ class OA(object):
                 for result in rep_services_results:
                     rep_results = {k: "{0}::{1}".format(rep_results.get(k, ""), result.get(k, "")).strip('::') for k in set(rep_results) | set(result)}
 
-                self._proxy_scores = [ conn + [ rep_results[conn[key]] ]   for conn in self._proxy_scores  ]
+                if rep_results:
+                    self._proxy_scores = [ conn + [ rep_results[conn[key]] ]   for conn in self._proxy_scores  ]
+                else:
+                    self._proxy_scores = [ conn + [""] for conn in self._proxy_scores  ]
         else:
             self._proxy_scores = [ conn + [""] for conn in self._proxy_scores  ]
 
 
-
-    def _add_severity(self):
-        # Add severity column
-        self._proxy_scores = [conn + [0] for conn in self._proxy_scores]
-
-
     def _add_iana(self):
 
         iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
         if os.path.isfile(iana_conf_file):
             iana_config  = json.loads(open(iana_conf_file).read())
             proxy_iana = IanaTransform(iana_config["IANA"])
-            proxy_rcode_index = self._conf["proxy_score_fields"]["respcode"]
-            self._proxy_scores = [ conn + [ proxy_iana.get_name(conn[proxy_rcode_index],"proxy_http_rcode")] for conn in self._proxy_scores ]
+            proxy_rcode_index = self._conf["proxy_score_fields"]["respcode"]            
+            self._proxy_scores = [ conn + [proxy_iana.get_name(conn[proxy_rcode_index],"proxy_http_rcode")] for conn in self._proxy_scores ]
         else:
             self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
 
@@ -225,96 +245,70 @@ class OA(object):
             nc_conf = json.loads(open(nc_conf_file).read())["NC"]
             proxy_nc = NetworkContext(nc_conf,self._logger)
             ip_dst_index = self._conf["proxy_score_fields"]["clientip"]
-            self._proxy_scores = [ conn + [proxy_nc.get_nc(conn[ip_dst_index])] for conn in self._proxy_scores ]
-
+            self._proxy_scores = [ conn + [proxy_nc.get_nc(conn[ip_dst_index])] for conn in self._proxy_scores ] 
         else:
             self._proxy_scores = [ conn + [""] for conn in self._proxy_scores ]
 
 
-    def _add_hash(self):
-        #A hash string is generated to be used as the file name for the edge files.
-        #These fields are used for the hash creation, so this combination of values is treated as
-        #a 'unique' connection
-        cip_index = self._conf["proxy_score_fields"]["clientip"]
-        uri_index = self._conf["proxy_score_fields"]["fulluri"]
-        tme_index = self._conf["proxy_score_fields"]["p_time"]
-
-        self._proxy_scores = [conn + [str( md5.new(str(conn[cip_index]) + str(conn[uri_index])).hexdigest() + str((conn[tme_index].split(":"))[0]) )] for conn in self._proxy_scores]
-
-
     def _get_oa_details(self):
 
         self._logger.info("Getting OA Proxy suspicious details")
         # start suspicious connects details process.
         p_sp = Process(target=self._get_suspicious_details)
         p_sp.start()
-
-        # p_sp.join()
+ 
 
     def _get_suspicious_details(self):
-        hash_list = []
+        uri_list = []
         iana_conf_file = "{0}/components/iana/iana_config.json".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
         if os.path.isfile(iana_conf_file):
             iana_config  = json.loads(open(iana_conf_file).read())
             proxy_iana = IanaTransform(iana_config["IANA"])
 
         for conn in self._proxy_scores:
-            conn_hash = conn[self._conf["proxy_score_fields"]["hash"]]
-            if conn_hash not in hash_list:
-                hash_list.append(conn_hash)
-                clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
-                fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
-                date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
-                if len(date) == 3:
-                    year=date[0]
-                    month=date[1].zfill(2)
-                    day=date[2].zfill(2)
-                    hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
-                    self._get_proxy_details(fulluri,clientip,conn_hash,year,month,day,hh,proxy_iana)
-
-
-    def _get_proxy_details(self,fulluri,clientip,conn_hash,year,month,day,hh,proxy_iana):
-
-        limit = 250
-        output_delimiter = '\t'
-        edge_file ="{0}/edge-{1}-{2}.tsv".format(self._data_path,clientip,conn_hash)
-        edge_tmp  ="{0}/edge-{1}-{2}.tmp".format(self._data_path,clientip,conn_hash)
-
-        if not os.path.isfile(edge_file):
-            proxy_qry = ("SELECT p_date, p_time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype, \
-                referer, uriport, serverip, scbytes, csbytes, fulluri FROM {0}.{1} WHERE y=\'{2}\' AND m=\'{3}\' AND d=\'{4}\' AND \
-                h=\'{5}\' AND fulluri =\'{6}\' AND clientip = \'{7}\' LIMIT {8};").format(self._db,self._table_name, year,month,day,hh,fulluri,clientip,limit)
-
-            # execute query
-            self._engine.query(proxy_qry,edge_tmp,output_delimiter)
-            # add IANA to results.
+            clientip = conn[self._conf["proxy_score_fields"]["clientip"]]
+            fulluri = conn[self._conf["proxy_score_fields"]["fulluri"]]
+            date=conn[self._conf["proxy_score_fields"]["p_date"]].split('-')
+            if len(date) == 3:
+                year=date[0]
+                month=date[1].zfill(2)
+                day=date[2].zfill(2)
+                hh=(conn[self._conf["proxy_score_fields"]["p_time"]].split(":"))[0]
+                self._get_proxy_details(fulluri,clientip,year,month,day,hh,proxy_iana)
+
+
+    def _get_proxy_details(self,fulluri,clientip,year,month,day,hh,proxy_iana):
+        limit = 250 
+        value_string = ""
+        
+        query_to_load =("""
+            SELECT p_date, p_time, clientip, host, webcat, respcode, reqmethod, useragent, resconttype,
+            referer, uriport, serverip, scbytes, csbytes, fulluri, {5} as hh
+            FROM {0}.{1} WHERE y='{2}' AND m='{3}' AND d='{4}' AND
+            h='{5}' AND fulluri='{6}' AND clientip='{7}' LIMIT {8};
+        """).format(self._db,self._table_name, year,month,day,hh,fulluri.replace("'","\\'"),clientip,limit)
+
+        detail_results = impala.execute_query(query_to_load)
+ 
+        if proxy_iana:
+             # add IANA to results.
             self._logger.info("Adding IANA translation to details results")
-            with open(edge_tmp) as proxy_details_csv:
-                rows = csv.reader(proxy_details_csv, delimiter=output_delimiter,quotechar='"')
-                next(proxy_details_csv)
-                update_rows = [[conn[0]] + [conn[1]] + [conn[2]] + [conn[3]] + [conn[4]] + [proxy_iana.get_name(conn[5],"proxy_http_rcode") if proxy_iana else conn[5]] + [conn[6]] + [conn[7]] + [conn[8]] + [conn[9]] + [conn[10]] + [conn[11]] + [conn[12]] + [conn[13]] + [conn[14]] if len(conn) > 0 else [] for conn in rows]
-                update_rows = filter(None, update_rows)
-                header = ["p_date","p_time","clientip","host","webcat","respcode","reqmethod","useragent","resconttype","referer","uriport","serverip","scbytes","csbytes","fulluri"]
-                update_rows.insert(0,header)
-
-		# due an issue with the output of the query.
-		update_rows = [ [ w.replace('"','') for w in l ] for l in update_rows ]
-	
-
-            # create edge file.
-            self._logger.info("Creating edge file:{0}".format(edge_file))
-            with open(edge_file,'wb') as proxy_details_edge:
-                writer = csv.writer(proxy_details_edge, quoting=csv.QUOTE_NONE, delimiter=output_delimiter)
-                if update_rows:
-                    writer.writerows(update_rows)
-                else:
-                    shutil.copy(edge_tmp,edge_file)
-
-            try:
-                os.remove(edge_tmp)
-            except OSError:
-                pass
+ 
+            updated_rows = [conn + (proxy_iana.get_name(conn[5],"proxy_http_rcode"),) for conn in detail_results]
+            updated_rows = filter(None, updated_rows)            
+        else:
+            updated_rows = [conn + ("") for conn in detail_results ]
+ 
+        for row in updated_rows:
+            value_string += str(tuple(item for item in row)) + ","     
+        
+        if value_string != "":  
+            query_to_insert=("""
+                INSERT INTO {0}.proxy_edge PARTITION (y={1}, m={2}, d={3}) VALUES ({4});
+            """).format(self._db,year, month, day, value_string[:-1])
 
+            impala.execute_query(query_to_insert) 
+ 
 
     def _ingest_summary(self): 
         # get date parameters.
@@ -328,44 +322,36 @@ class OA(object):
         result_rows = []        
         df_filtered =  pd.DataFrame()
 
-        ingest_summary_file = "{0}/is_{1}{2}.csv".format(self._ingest_summary_path,yr,mn)			
-        ingest_summary_tmp = "{0}.tmp".format(ingest_summary_file)
-
-        if os.path.isfile(ingest_summary_file):
-        	df = pd.read_csv(ingest_summary_file, delimiter=',')
-            #discards previous rows from the same date
-        	df_filtered = df[df['date'].str.contains("{0}-{1}-{2}".format(yr, mn, dy)) == False] 
-        else:
-        	df = pd.DataFrame()
-            
         # get ingest summary.
-        ingest_summary_qry = ("SELECT p_date, p_time, COUNT(*) as total "
-                                    " FROM {0}.{1}"
-                                    " WHERE y='{2}' AND m='{3}' AND d='{4}' "
-                                    " AND p_date IS NOT NULL AND p_time IS NOT NULL " 
-                                    " AND clientip IS NOT NULL AND p_time != '' "
-                                    " AND host IS NOT NULL AND fulluri IS NOT NULL "
-                                    " GROUP BY p_date, p_time;") 
-
-        ingest_summary_qry = ingest_summary_qry.format(self._db,self._table_name, yr, mn, dy)
-        results_file = "{0}/results_{1}.csv".format(self._ingest_summary_path,self._date)        
-        self._engine.query(ingest_summary_qry,output_file=results_file,delimiter=",")
+
+        query_to_load=("""
+                SELECT p_date, p_time, COUNT(*) as total
+                FROM {0}.{1} WHERE y='{2}' AND m='{3}' AND d='{4}'
+                AND p_date IS NOT NULL AND p_time IS NOT NULL
+                AND clientip IS NOT NULL AND p_time != ''
+                AND host IS NOT NULL AND fulluri IS NOT NULL
+                GROUP BY p_date, p_time;
+        """).format(self._db,self._table_name, yr, mn, dy)
         
-        if os.path.isfile(results_file):
-            df_results = pd.read_csv(results_file, delimiter=',')  
-            
+        results = impala.execute_query(query_to_load) 
+ 
+        if results:
+            df_results = as_pandas(results)
             #Forms a new dataframe splitting the minutes from the time column/
             df_new = pd.DataFrame([["{0} {1}:{2}".format(val['p_date'], val['p_time'].split(":")[0].zfill(2), val['p_time'].split(":")[1].zfill(2)), int(val['total']) if not math.isnan(val['total']) else 0 ] for key,val in df_results.iterrows()],columns = ingest_summary_cols)
-            
+            value_string = ''
             #Groups the data by minute 
             sf = df_new.groupby(by=['date'])['total'].sum()
             df_per_min = pd.DataFrame({'date':sf.index, 'total':sf.values})
             
-            df_final = df_filtered.append(df_per_min, ignore_index=True)
-            df_final.to_csv(ingest_summary_tmp,sep=',', index=False)
-
-            os.remove(results_file)
-            os.rename(ingest_summary_tmp,ingest_summary_file)
+            df_final = df_filtered.append(df_per_min, ignore_index=True).to_records(False,False) 
+            if len(df_final) > 0:
+                query_to_insert=("""
+                    INSERT INTO {0}.proxy_ingest_summary PARTITION (y={1}, m={2}, d={3}) VALUES {4};
+                """).format(self._db, yr, mn, dy, tuple(df_final))
+
+                impala.execute_query(query_to_insert) 
+                
         else:
             self._logger.info("No data found for the ingest summary")
-        
\ No newline at end of file
+        

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/start_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/start_oa.py b/spot-oa/oa/start_oa.py
index b604aab..660814f 100644
--- a/spot-oa/oa/start_oa.py
+++ b/spot-oa/oa/start_oa.py
@@ -20,6 +20,7 @@
 import argparse
 import os
 import sys
+sys.path.append("../")
 import logging
 
 from utils import Util
@@ -39,43 +40,43 @@ def main():
     start_oa(args)
 
 def start_oa(args):
-    
-    # setup the main logger for all the OA process.    
+
+    # setup the main logger for all the OA process.
     logger = Util.get_logger('OA',create_file=False)
 
-    logger.info("-------------------- STARTING OA ---------------------")   
-    validate_parameters_values(args,logger)   
+    logger.info("-------------------- STARTING OA ---------------------")
+    validate_parameters_values(args,logger)
 
     # create data type instance.
     module = __import__("{0}.{0}_oa".format(args.type),fromlist=['OA'])
-   
-    # start OA.   
+
+    # start OA.
     oa_process = module.OA(args.date,args.limit,logger)
     oa_process.start()
-  
+
 def validate_parameters_values(args,logger):
-    
+
     logger.info("Validating input parameter values")
 
     #date.
-    is_date_ok = True if len(args.date) == 8 else False    
+    is_date_ok = True if len(args.date) == 8 else False
 
     # type
     dirs = os.walk(script_path).next()[1]
     is_type_ok = True if args.type in dirs else False
-   
-    #limit    
+
+    #limit
     try:
         int(args.limit)
         is_limit_ok = True
     except ValueError:
         is_limit_ok = False
-      
-    if not is_date_ok: logger.error("date parameter is not correct, please validate it") 
+
+    if not is_date_ok: logger.error("date parameter is not correct, please validate it")
     if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
     if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
     if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1)
-   
+
 
 if __name__=='__main__':
-    main()
\ No newline at end of file
+    main()

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/oa/utils.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/utils.py b/spot-oa/oa/utils.py
index 99b006e..2bed10e 100644
--- a/spot-oa/oa/utils.py
+++ b/spot-oa/oa/utils.py
@@ -121,6 +121,18 @@ class Util(object):
 			writer.writerows(content)
 
 
+	@classmethod
+    	def cast_val(self,value):
+       	    try: 
+            	val = int(value) 
+            except:
+            	try:
+                    val = float(value) 
+            	except:
+                    val = str(value) 
+            return val    
+
+
 class SecHead(object):
     def __init__(self, fp):
         self.fp = fp

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/requirements.txt
----------------------------------------------------------------------
diff --git a/spot-oa/requirements.txt b/spot-oa/requirements.txt
index 312360b..9f3afb8 100644
--- a/spot-oa/requirements.txt
+++ b/spot-oa/requirements.txt
@@ -5,7 +5,22 @@ Tld == 0.7.6
 
 # spot-ui dependencies
 jsonschema
-ipython == 3.2.0
 pyzmq
 jinja2
 tornado
+mistune
+pygments
+terminado
+ipython == 3.2.1
+
+# GraphQL API dependencies
+flask
+flask-graphql
+graphql-core
+urllib3
+
+# API Resources
+setuptools>=3.4.4
+thrift==0.9.3
+impyla
+hdfs

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/runIpython.sh
----------------------------------------------------------------------
diff --git a/spot-oa/runIpython.sh b/spot-oa/runIpython.sh
index 04a09d9..38a4121 100755
--- a/spot-oa/runIpython.sh
+++ b/spot-oa/runIpython.sh
@@ -1,5 +1,4 @@
-#!/bin/sh
-
+#!/bin/bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one or more
 # contributor license agreements.  See the NOTICE file distributed with
@@ -16,5 +15,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-ipython notebook --no-mathjax --profile=ia --port=8889 --ip=0.0.0.0 --no-browser '--NotebookApp.extra_static_paths=["ui/ipython/"]'> ipython.out 2>&1&
+pushd $(dirname $0) > /dev/null
+SPOT_OA_DIR=$(pwd)
+popd > /dev/null
+IPYTHONDIR="$SPOT_OA_DIR/ipython/"
+export IPYTHONDIR
+ipython notebook --profile=spot > ipython.out 2>&1&

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/ui/README.md b/spot-oa/ui/README.md
index 4b35c86..d068b39 100755
--- a/spot-oa/ui/README.md
+++ b/spot-oa/ui/README.md
@@ -68,12 +68,15 @@ Provides tools for interactive visualization, noise filters, white listing, and
           + [Suspicious](#suspicious-4)
           + [Storyboard](#storyboard-4)
           + [Ingest Summary](#ingest-summary-2)
+          + [Notebook](#notebook-1)
         * [DNS Stores](#dns-stores)
           + [Suspicious](#suspicious-5)
           + [Storyboard](#storyboard-5)
+          + [Notebook](#notebook-2)
         * [Proxy Stores](#proxy-stores)
           + [Suspicious](#suspicious-6)
           + [Storyboard](#storyboard-6)
+          + [Notebook](#notebook-3)
 
 ## Intended Audience
 
@@ -91,7 +94,7 @@ Here you will find useful information to get you started on how to contribute to
 
 ## Technical Documentation
 
-Our UI is build on top of Flux application architecture principles, having an understanding of this pattern is highly recommended, for more information about Flux, please go to [Flux web page](https://facebook.github.io/flux/docs/overview.html).
+Our UI is built on top of Flux application architecture principles, having an understanding of this pattern is highly recommended, for more information about Flux, please go to [Flux web page](https://facebook.github.io/flux/docs/overview.html).
 
 ### ReactJS + Flux
 
@@ -104,12 +107,13 @@ For more information about ReactJS and Flux, please go to:
 
 From now on we assume you are familiar with ReactJS+Flux applications.
 
-For every path found on this document, "SPOT" refers to the path where Spot UI is intalled and "PIPELINE" the pipeline subfolder you want to work on.
+For every path found on this document, "SPOT" refers to the path where Spot UI is installed and "PIPELINE" the pipeline subfolder you want to work on.
 
 #### Development/Debugging process
 
 1. Install Spot UI. Follow this [guide](INSTALL.md#install-spot-ui).
 2. Run Spot UI. Follow this [guide](INSTALL.md#how-to-run-spot-ui).
+    1. Set SPOT_DEV env var to 1 to enable GraphiQL UI and run ipython in debug mode.
 3. Start watching for code changes
     1. $ cd SPOT/ui/PIPELINE/
     2. Watch one of the following modules
@@ -1034,6 +1038,13 @@ _SPOT/ui/js/flow/stores/_
 
     Extends [RestStore.js](#reststore.js)
 
+###### Notebook
+
+spot-oa/ui/flow/js/stores/NotebookStore.js
+
+spot-oa/ui/flow/js/components/ScoreNotebook.react.js
+
+
 ##### DNS Stores
 
 _SPOT/ui/js/dns/stores/_
@@ -1095,6 +1106,12 @@ _SPOT/ui/js/dns/stores/_
     > - SELECT_COMMENT
     > - RELOAD_COMMENTS
 
+###### Notebook
+
+spot-oa/ui/dns/js/stores/NotebookStore.js
+
+spot-oa/ui/dns/js/components/ScoreNotebook.react.js
+
 ##### Proxy Stores
 
 _SPOT/ui/js/proxy/_
@@ -1153,3 +1170,9 @@ _SPOT/ui/js/proxy/_
     > - UPDATE_DATE
     > - SELECT_COMMENT
     > - RELOAD_COMMENTS
+
+###### Notebook
+
+spot-oa/ui/proxy/js/stores/NotebookStore.js
+
+spot-oa/ui/proxy/js/components/ScoreNotebook.react.js
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/bootstrap-spot.min.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/bootstrap-spot.min.css b/spot-oa/ui/css/bootstrap-spot.min.css
index 6d05096..cbd2297 100644
--- a/spot-oa/ui/css/bootstrap-spot.min.css
+++ b/spot-oa/ui/css/bootstrap-spot.min.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 /*! Generated by Live LESS Theme Customizer */
 .label,sub,sup{vertical-align:baseline}
 body,figure{margin:0}

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/dendrogram.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/dendrogram.css b/spot-oa/ui/css/dendrogram.css
index 06782df..0395c08 100644
--- a/spot-oa/ui/css/dendrogram.css
+++ b/spot-oa/ui/css/dendrogram.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
     .dendrogram {
       overflow-y: auto;
       overflow-x: auto;

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/css/main.css
----------------------------------------------------------------------
diff --git a/spot-oa/ui/css/main.css b/spot-oa/ui/css/main.css
index 00368ca..8421f13 100755
--- a/spot-oa/ui/css/main.css
+++ b/spot-oa/ui/css/main.css
@@ -1,3 +1,19 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
 /* All devices */
 .spot-maximized {
     width: 100% !important;
@@ -47,6 +63,7 @@
 .spot-frame > .spot-frame-content > .panel > .panel-body > .panel-body-container {
     width: 100%;
     overflow-x: auto;
+    height: 1px;
 }
 
 .spot-frame .panel-body-container.container-box > * {
@@ -57,6 +74,39 @@
     border: none;
 }
 
+.sortable {
+    display: flex;
+    flex-flow: column;
+    height: 100%;
+}
+
+.overflowing {
+    overflow-y: scroll;
+}
+
+#accordion {
+    height: 85%;
+}
+
+.fit {
+    height: 50%;
+}
+
+::-webkit-scrollbar {
+    width: 7px;
+    height: 7px;
+}
+
+::-webkit-scrollbar-track {
+    -webkit-box-shadow: inset 0 0 6px rgba(0,0,0,0.3);
+    border-radius: 10px;
+}
+
+::-webkit-scrollbar-thumb {
+    border-radius: 10px;
+    -webkit-box-shadow: inset 0 0 10px rgba(0,0,0,0.5);
+}
+
 /* Extra small devices (phones, less than 768px) */
 
 /* Small devices (tablets, 768px and up) */
@@ -291,3 +341,24 @@ prevent the active panel from collapsing
 .spot-text-wrapper.spot-text-lg {
     width: 18em;
 }
+
+/*  GLYPHICON INPUT  */
+
+.inner-addon {
+  position: relative;
+}
+
+.inner-addon .glyphicon {
+  position: absolute;
+  padding: 10px;
+  pointer-events: none;
+}
+
+.left-addon .glyphicon  { left:  0px;}
+.right-addon .glyphicon { right: 0px;}
+
+.right-addon input { padding-right: 30px; }
+
+.margin-up-down {
+  padding: 20px 0px;
+}

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/ipython_notebook.html b/spot-oa/ui/dns/ipython_notebook.html
new file mode 100644
index 0000000..21e4572
--- /dev/null
+++ b/spot-oa/ui/dns/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~    http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<html>
+<head>
+    <title>DNS :: Advanced mode</title>
+
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+    <!--Bootstrap styles-->
+    <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+    <!--Font awesome ICONS-->
+    <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+    <!--Bootstrap Date picker styles-->
+    <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+    <!-- Spot styles -->
+    <link rel="stylesheet" type="text/css" href="../css/main.css" />
+    <!-- Sweetalert2 -->
+    <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+    <!-- Favicon -->
+    <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+    <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+    <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+    <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+    <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+    <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+    <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+    <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+    <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+    <link rel="icon" type="image/png" sizes="192x192"  href="../images/favicon/android-icon-192x192.png">
+    <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+    <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+    <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+    <link rel="manifest" href="../images/favicon/manifest.json">
+    <meta name="msapplication-TileColor" content="#ffffff">
+    <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+    <meta name="theme-color" content="#ffffff">
+
+    <style>
+        .spot-row {
+            height: 100%;
+        }
+
+        td.srcIP_rep, td.dstIP_rep {
+            white-space: nowrap;
+        }
+    </style>
+</head>
+<body>
+    <nav id="spot-nav" class="navbar navbar-default">
+        <div class="container-fluid">
+            <!-- App name and toggle get grouped for better mobile display -->
+            <div class="navbar-header">
+                <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+                    <span class="sr-only">Toggle navigation</span>
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                </button>
+                <span class="navbar-brand">Apache Spot :: DNS :: Advanced mode</span>
+            </div>
+            <!-- Collect the nav links, forms, and other content for toggling -->
+            <div class="collapse navbar-collapse" id="main-menu">
+              <!-- Main Menu -->
+            </div>
+            <div id="search-box" class="row text-right">
+                <!--Tools Buttons-->
+                <div id="nav_form" class="col-md-12">
+                  <!-- Search form placeholder -->
+                </div>
+            </div> <!-- /Tools Buttons-->
+        </div>
+    </nav>
+    <div id="spot-content-wrapper" class="container-fluid">
+        <!-- Main Content Placeholder -->
+    </div>
+
+    <!-- SCRIPTS -->
+    <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+    <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+    <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+    <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+    <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+    <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+    <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+    <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+    <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+    <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+    <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js b/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
index 462e040..ad40124 100755
--- a/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsDendrogramPanel.react.js
@@ -1,5 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 const React = require('react');
 
 const ContentLoaderMixin = require('../../../js/components/ContentLoaderMixin.react');
@@ -29,7 +43,7 @@ const DetailsDendrogramPanel = React.createClass({
 
             state.data = {
                 id: 'root',
-                name: DendrogramStore.getSrcIp(),
+                name: DendrogramStore.getClientIp(),
                 children: []
             };
 
@@ -45,7 +59,7 @@ const DetailsDendrogramPanel = React.createClass({
                 };
                 state.data.children.push(childNode);
 
-                answers = item.dns_a.split('|');
+                answers = item.dns_a;
 
                 if (answers.length) {
                     let childId = 0;

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsPanel.react.js b/spot-oa/ui/dns/js/components/DetailsPanel.react.js
index 550c516..15fc621 100755
--- a/spot-oa/ui/dns/js/components/DetailsPanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
 var React = require('react');
 

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js b/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
index 59c5e23..0ed697e 100755
--- a/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
+++ b/spot-oa/ui/dns/js/components/DetailsTablePanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
 var React = require('react');
 
@@ -30,7 +45,6 @@ var DetailsTablePanel = React.createClass({
   },
   _render_dns_a_cell: function (answers)
   {
-    answers = (answers || "").split('|');
     answers = answers.map(function (answer, idx)
     {
         return (
@@ -42,9 +56,6 @@ var DetailsTablePanel = React.createClass({
 
     return answers;
   },
-  _render_dns_qry_class_cell: false,
-  _render_dns_qry_rcode_cell: false,
-  _render_dns_qry_type_cell: false,
   _onChange: function ()
   {
     this.replaceState(DetailsStore.getData());

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/FilterInput.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/FilterInput.react.js b/spot-oa/ui/dns/js/components/FilterInput.react.js
index 666fc3e..3599603 100755
--- a/spot-oa/ui/dns/js/components/FilterInput.react.js
+++ b/spot-oa/ui/dns/js/components/FilterInput.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
 var React = require('react');
 

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js b/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
index da17da7..ca322cc 100755
--- a/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
+++ b/spot-oa/ui/dns/js/components/IncidentProgressionPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
 const React = require('react');
 
@@ -8,8 +23,8 @@ const DendrogramMixin = require('../../../js/components/DendrogramMixin.react');
 const IncidentProgressionStore = require('../stores/IncidentProgressionStore');
 
 const fieldMapper = {
-    ip_dst: 'dns_qry_name',
-    dns_qry_name: 'ip_dst'
+    clientIp: 'dnsQuery',
+    dnsQuery: 'clientIp'
 };
 
 const IncidentProgressionPanel = React.createClass({

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js b/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
index 7328c76..d2ab8bd 100755
--- a/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
+++ b/spot-oa/ui/dns/js/components/NetworkViewPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
 const React = require('react');
 
@@ -46,7 +61,7 @@ function getNodesFromData(data) {
                 nodes[id] = {
                     id: id,
                     label: item[field],
-                    internalIp: field==='tld',
+                    internalIp: field==='tld' ? 1 : 0,
                     hits: 1
                 };
             }

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/ScoreNotebook.react.js b/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
new file mode 100644
index 0000000..e6c5259
--- /dev/null
+++ b/spot-oa/ui/dns/js/components/ScoreNotebook.react.js
@@ -0,0 +1,206 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+var React = require('react');
+
+var SuspiciousStore = require('../stores/SuspiciousStore');
+var NotebookStore = require('../stores/NotebookStore');
+var EdInActions = require('../../../js/actions/EdInActions');
+var SelectInput = require('../../../js/components/scoring/SelectInput.react');
+var FilterSelectInput = require('../../../js/components/scoring/FilterSelectInput.react');
+var ButtonsScoring = require('../../../js/components/scoring/ButtonsScoring.react');
+var RatingInput = require('../../../js/components/scoring/RatingInput.react');
+var SearchGlobalInput = require('../../../js/components/scoring/SearchGlobalInput.react');
+var ScoreMessage = require('../../../js/components/scoring/ScoreMessage.react');
+
+var SpotUtils = require('../../../js/utils/SpotUtils');
+
+var ScoreNotebook = React.createClass({
+  // mixins: [GridPanelMixin],
+  emptySetMessage: 'There is no data available for selected date.',
+  propTypes: {
+    date: React.PropTypes.string.isRequired,
+  },
+  getInitialState: function () {
+    return {
+      scoredEmelents: []
+    };
+  },
+  componentDidMount: function() {
+    SuspiciousStore.addChangeDataListener(this._onChange);
+    NotebookStore.addChangeDataListener(this._onChange);
+    this.setState({size: NotebookStore.completeClass})
+  },
+  componentWillUnmount: function () {
+    SuspiciousStore.removeChangeDataListener(this._onChange);
+    NotebookStore.addChangeDataListener(this._onChange);
+  },
+  render: function () {
+    var content, state, data, cssCls, dnsNameArr = [], dstIpArr = [];
+    state = this.state || {};
+
+    if (state.error)
+    {
+      content = (
+        <div className="text-center text-danger">
+          {state.error}
+        </div>
+      );
+    }
+    else if (state.loading)
+    {
+      content = (
+        <div className="spot-loader">
+          Loading <span className="spinner"></span>
+        </div>
+      );
+    }
+    else if (!state.data || state.data.length === 0)
+    {
+      content = (
+        <div className="text-center">
+          {this.emptySetMessage || ''}
+        </div>
+      );
+    }
+    else
+    {
+      state.data.map((obj) => {
+        if(dstIpArr.indexOf(obj.ip_dst) === -1) {
+          dstIpArr.push(obj.ip_dst);
+        }
+        if(dnsNameArr.indexOf(obj.dns_qry_name) === -1) {
+          dnsNameArr.push(obj.dns_qry_name);
+        }
+      });
+
+      data = [
+              {value: 1, name: 'High', radioName: 'optradio', selected: true},
+              {value: 2, name: 'Medium', radioName: 'optradio', selected: false},
+              {value: 3, name: 'Low', radioName: 'optradio', selected: false}
+            ];
+
+      content = (
+        <div>
+            <div className="margin-up-down">
+              <SearchGlobalInput col="6" maxlength="255"/>
+              <RatingInput data={data} col="6"/>
+            </div>
+            <div className="margin-up-down">
+              <ButtonsScoring name="Score" action="score" onChange={this.score.bind(this)} col="3"/>
+              <ButtonsScoring name="Save" action="save" onChange={this.save.bind(this)} col="3"/>
+              <ButtonsScoring name="Reset Scoring" action="reset" onChange={this.reset.bind(this)} col="3"/>
+            </div>
+            <div className="margin-up-down">
+              <FilterSelectInput nameBox="Client Ip" idSelect="#dstIp" idInput="dstIpIn" col="3"/>
+              <FilterSelectInput nameBox="Query" idSelect="#query" idInput="queryIn" col="9"/>
+            </div>
+            <div className="margin-up-down">
+              <SelectInput title="Client IP" who="dstIp" options={dstIpArr} col="3"/>
+              <SelectInput title="Query" who="query" options={dnsNameArr} col="9"/>
+            </div>
+            <div className="margin-up-down">
+              <ScoreMessage who="scoreMsg"/>
+            </div>
+          </div>
+        );
+    }
+    cssCls = this.state.size ? 'col-md-6 col-lg-6 col-xs-12' : 'col-md-offset-3 col-lg-offset-3 col-md-6 col-lg-6 col-xs-12';
+
+    return(
+      <div className={cssCls + ' spot-frame'}>
+        {content}
+      </div>
+    )
+  },
+  _onChange: function() {
+    const data = SuspiciousStore.getData();
+    this.setState(data);
+  },
+  reset: function() {
+    swal({
+      title: 'Are you sure?',
+      text: "You won't be able to revert this!",
+      type: 'warning',
+      showCancelButton: true,
+      confirmButtonColor: '#3085d6',
+      cancelButtonColor: '#d33',
+      confirmButtonText: 'Yes, reset all!'
+    }).then(() => {
+      EdInActions.resetScoring(SpotUtils.getCurrentDate());
+      swal({
+        title: 'Done!',
+        text: "All scores have been reset.",
+        type: 'success',
+        showCancelButton: false,
+        confirmButtonColor: '#3085d6',
+        cancelButtonColor: '#d33',
+        confirmButtonText: 'Ok!'
+      }).then(() => {
+        this.setState({loading: true});
+      });
+    });
+  },
+  save: function() {
+    let variables = [];
+
+    if (this.state.scoredEmelents.length === 0) {
+      swal('Warning.','You need to score at least 1 threat.','warning');
+    } else {
+      this.state.scoredEmelents.map((row) => {
+        variables.push({
+          'date': SpotUtils.getCurrentDate(),
+          'clientIp': row[0] || null,
+          'dnsQuery':  row[1] || null,
+          'score': row[2]
+        });
+      });
+
+      EdInActions.saveScoring(variables);
+      $('#scoreMsg').addClass("hidden");
+      this.setState({scoredEmelents: [], loading: true});
+    }
+  },
+  score: function() {
+    //this should be changed to take all data at the time, time needed.
+    let dataScored = this.state.scoredEmelents || [];
+    let quickIpScoring = document.getElementById('globalTxt').value;
+    let query   = document.getElementById('query').value;
+    let dstIp   = document.getElementById('dstIp').value;
+    let rating  = $('input[name="optradio"]:checked').val();
+
+    //callback from the father
+    if(quickIpScoring !== '') {
+      dataScored.push([dstIp, quickIpScoring, rating]);
+    } else {
+      dataScored.push([dstIp, query, rating]);
+    }
+
+    this.removeSelected([dstIp, quickIpScoring,query]);
+    $('#scoreMsg').removeClass("hidden");
+    this.setState({scoredEmelents: dataScored});
+  },
+  removeSelected: function(data) {
+    //when an user score, all elements selected need to be removed.
+    data.map((element) => data.map((e) => e !== '' ? $(`option[value="${e}"]`).remove() : ''));
+    $(".select-picker, #globalTxt").val('');
+  }
+
+});
+
+
+module.exports = ScoreNotebook;

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js b/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
index 951319e..a2d106f 100755
--- a/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
+++ b/spot-oa/ui/dns/js/components/SuspiciousPanel.react.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
 
 var React = require('react');
 
@@ -26,28 +41,6 @@ var SuspiciousPanel = React.createClass({
       </p>
     );
   },
-  /**
-    * Answers cell can have multiple pipe separated values. To allow
-    * proper displaying of information lets place each answer inside
-    * a block element let browsers decide how to display them.
-    */
-  _render_dns_a_cell: function (answers, item, idx)
-  {
-    var cellBody;
-
-    answers = (answers || []).split('|');
-
-    cellBody = answers.map(function (answer, i)
-    {
-      return (
-        <div key={'answer_' + idx + '_' + i}>
-          {answer}
-        </div>
-      );
-    }.bind(this));
-
-    return cellBody;
-  },
   _render_ip_dst_cell: function (ip_dst, item, idx)
   {
     var ip_dst_info, iconClass;

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/constants/DnsConstants.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/constants/DnsConstants.js b/spot-oa/ui/dns/js/constants/DnsConstants.js
deleted file mode 100755
index 9e269fa..0000000
--- a/spot-oa/ui/dns/js/constants/DnsConstants.js
+++ /dev/null
@@ -1,12 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var DnsConstants = {
-  // API URLS
-  API_SUSPICIOUS: '../../data/dns/${date}/dns_scores.csv',
-  API_DETAILS: '../../data/dns/${date}/edge-${dns_qry_name}_${time}.csv',
-  API_VISUAL_DETAILS: '../../data/dns/${date}/dendro-${ip_dst}.csv',
-  API_COMMENTS: '../../data/dns/${date}/threats.csv',
-  API_INCIDENT_PROGRESSION: '../../data/dns/${date}/threat-dendro-${id}.csv'
-};
-
-module.exports = DnsConstants;

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/notebooks.js b/spot-oa/ui/dns/js/notebooks.js
new file mode 100644
index 0000000..81d923e
--- /dev/null
+++ b/spot-oa/ui/dns/js/notebooks.js
@@ -0,0 +1,72 @@
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+  <MainMenu />,
+  document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+    (
+      <form className="form-inline">
+        <div className="form-group">
+          <label htmlFor="dataDatePicker">Data Date:</label>
+          <div className="input-group input-group-xs">
+            <DateInput id="dataDatePicker" />
+            <div className="input-group-addon">
+              <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+            </div>
+          </div>
+        </div>
+      </form>
+    ),
+    document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+  <div id="spot-content">
+    <PanelRow maximized>
+      <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+        <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="dns/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+      </Panel>
+    </PanelRow>
+  </div>,
+  document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/dns/js/stores/CommentsStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/stores/CommentsStore.js b/spot-oa/ui/dns/js/stores/CommentsStore.js
index 291af68..47e2eea 100755
--- a/spot-oa/ui/dns/js/stores/CommentsStore.js
+++ b/spot-oa/ui/dns/js/stores/CommentsStore.js
@@ -1,33 +1,73 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-var d3 = require('d3');
-
-var DnsConstants = require('../constants/DnsConstants');
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var RestStore = require('../../../js/stores/RestStore');
-
-var CommentsStore = assign(new RestStore(DnsConstants.API_COMMENTS), {
-  _parser: d3.dsv('|', 'text/plain'),
-  errorMessages: {
-    404: 'Please choose a different date, no comments have been found'
-  },
-  setDate: function (date)
-  {
-    this.setEndpoint(DnsConstants.API_COMMENTS.replace('${date}', date.replace(/-/g, '')));
-  }
-});
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements.  See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License.  You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+
+class CommentsStore extends ObservableGraphQLStore {
+    getQuery() {
+        return `
+            query($date:SpotDateType) {
+                dns {
+                    threats{
+                        comments(date:$date) {
+                            title
+                            summary: text
+                            ...QueryComment
+                            ...ClientIpComment
+                        }
+                    }
+                }
+            }
+
+            fragment QueryComment on DnsQueryCommentType {
+                dnsQuery
+            }
+
+            fragment ClientIpComment on DnsClientIpCommentType {
+                clientIp
+            }
+        `;
+    }
+
+    unboxData(data) {
+        return data.dns.threats.comments;
+    }
+
+    setDate(date) {
+        this.setVariable(DATE_VAR, date);
+    }
+}
+
+const cs = new CommentsStore();
 
 SpotDispatcher.register(function (action) {
   switch (action.actionType) {
     case SpotConstants.UPDATE_DATE:
-      CommentsStore.setDate(action.date);
+      cs.setDate(action.date);
       break;
     case SpotConstants.RELOAD_COMMENTS:
-      CommentsStore.reload();
+      cs.sendQuery();
       break;
   }
 });
 
-module.exports = CommentsStore;
+module.exports = cs;