You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by le...@apache.org on 2017/06/23 22:40:51 UTC
[1/5] incubator-spot git commit: Updated the main menu and created
html pages and containers to hold the advanced mode notebook
Repository: incubator-spot
Updated Branches:
refs/heads/apache/SPOT-35_graphql_api [created] e92b205d9
Updated the main menu and created html pages and containers to hold the advanced mode notebook
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/e92b205d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/e92b205d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/e92b205d
Branch: refs/heads/apache/SPOT-35_graphql_api
Commit: e92b205d912bc340ab86c82f968f49093ce31b1b
Parents: 0eac32f
Author: LedaLima <le...@apache.org>
Authored: Thu Jun 22 13:21:52 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Fri Jun 23 17:39:07 2017 -0500
----------------------------------------------------------------------
spot-oa/ui/dns/ipython_notebook.html | 108 +++++++++++++++++++
spot-oa/ui/dns/js/notebooks.js | 57 ++++++++++
spot-oa/ui/dns/package.json | 6 +-
spot-oa/ui/flow/ipython_notebook.html | 108 +++++++++++++++++++
spot-oa/ui/flow/js/notebooks.js | 57 ++++++++++
spot-oa/ui/flow/package.json | 6 +-
.../js/components/IPythonNotebookPanel.react.js | 8 +-
spot-oa/ui/js/menu/menu.js | 8 +-
spot-oa/ui/package.json | 2 +-
spot-oa/ui/proxy/ipython_notebook.html | 108 +++++++++++++++++++
spot-oa/ui/proxy/js/notebooks.js | 57 ++++++++++
spot-oa/ui/proxy/package.json | 6 +-
12 files changed, 517 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/dns/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/ipython_notebook.html b/spot-oa/ui/dns/ipython_notebook.html
new file mode 100644
index 0000000..21e4572
--- /dev/null
+++ b/spot-oa/ui/dns/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>DNS :: Advanced mode</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Font awesome ICONS-->
+ <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+
+ <style>
+ .spot-row {
+ height: 100%;
+ }
+
+ td.srcIP_rep, td.dstIP_rep {
+ white-space: nowrap;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: DNS :: Advanced mode</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+ <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/dns/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/js/notebooks.js b/spot-oa/ui/dns/js/notebooks.js
new file mode 100644
index 0000000..32c8392
--- /dev/null
+++ b/spot-oa/ui/dns/js/notebooks.js
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+ (
+ <form className="form-inline">
+ <div className="form-group">
+ <label htmlFor="dataDatePicker">Data Date:</label>
+ <div className="input-group input-group-xs">
+ <DateInput id="dataDatePicker" />
+ <div className="input-group-addon">
+ <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+ </div>
+ </div>
+ </div>
+ </form>
+ ),
+ document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+ <div id="spot-content">
+ <PanelRow maximized>
+ <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+ <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="dns/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+ </Panel>
+ </PanelRow>
+ </div>,
+ document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/dns/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/dns/package.json b/spot-oa/ui/dns/package.json
index d3f31bb..f57d258 100755
--- a/spot-oa/ui/dns/package.json
+++ b/spot-oa/ui/dns/package.json
@@ -6,10 +6,12 @@
"watch-suspicious": "watchify js/suspicious.js -o js/suspicious.bundle.min.js -v -d",
"watch-threat-investigation": "watchify js/threat-investigation.js -o js/threat-investigation.bundle.min.js -v -d",
"watch-storyboard": "watchify js/storyboard.js -o js/storyboard.bundle.min.js -v -d",
- "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard",
+ "watch-notebooks": "watchify js/notebooks.js -o js/notebooks.bundle.min.js -v -d",
+ "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard && npm run build-notebooks",
"build-suspicious": "browserify js/suspicious.js | uglifyjs -cm > js/suspicious.bundle.min.js",
"build-threat-investigation": "browserify js/threat-investigation.js | uglifyjs -cm > js/threat-investigation.bundle.min.js",
- "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js"
+ "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js",
+ "build-notebooks": "browserify js/notebooks.js | uglifyjs -cm > js/notebooks.bundle.min.js"
},
"browserify": {
"transform": [
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/flow/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/ipython_notebook.html b/spot-oa/ui/flow/ipython_notebook.html
new file mode 100644
index 0000000..17c59c8
--- /dev/null
+++ b/spot-oa/ui/flow/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>Netflow :: Advanced mode</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Font awesome ICONS-->
+ <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+
+ <style>
+ .spot-row {
+ height: 100%;
+ }
+
+ td.srcIP_rep, td.dstIP_rep {
+ white-space: nowrap;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: Netflow :: Advanced mode</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+ <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/flow/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/js/notebooks.js b/spot-oa/ui/flow/js/notebooks.js
new file mode 100644
index 0000000..4720361
--- /dev/null
+++ b/spot-oa/ui/flow/js/notebooks.js
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+ (
+ <form className="form-inline">
+ <div className="form-group">
+ <label htmlFor="dataDatePicker">Data Date:</label>
+ <div className="input-group input-group-xs">
+ <DateInput id="dataDatePicker" />
+ <div className="input-group-addon">
+ <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+ </div>
+ </div>
+ </div>
+ </form>
+ ),
+ document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+ <div id="spot-content">
+ <PanelRow maximized>
+ <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+ <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="flow/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+ </Panel>
+ </PanelRow>
+ </div>,
+ document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/flow/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/flow/package.json b/spot-oa/ui/flow/package.json
index 62f47ae..f5d1704 100755
--- a/spot-oa/ui/flow/package.json
+++ b/spot-oa/ui/flow/package.json
@@ -6,10 +6,12 @@
"watch-suspicious": "watchify js/suspicious.js -o js/suspicious.bundle.min.js -v -d",
"watch-threat-investigation": "watchify js/threat-investigation.js -o js/threat-investigation.bundle.min.js -v -d",
"watch-storyboard": "watchify js/storyboard.js -o js/storyboard.bundle.min.js -v -d",
- "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard",
+ "watch-notebooks": "watchify js/notebooks.js -o js/notebooks.bundle.min.js -v -d",
+ "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard && npm run build-notebooks",
"build-suspicious": "browserify js/suspicious.js | uglifyjs -cm > js/suspicious.bundle.min.js",
"build-threat-investigation": "browserify js/threat-investigation.js | uglifyjs -cm > js/threat-investigation.bundle.min.js",
- "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js"
+ "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js",
+ "build-notebooks": "browserify js/notebooks.js | uglifyjs -cm > js/notebooks.bundle.min.js"
},
"browserify": {
"transform": [
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/components/IPythonNotebookPanel.react.js b/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
index 424d6ce..fb89e36 100755
--- a/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
+++ b/spot-oa/ui/js/components/IPythonNotebookPanel.react.js
@@ -5,6 +5,7 @@ var React = require('react');
var SpotActions = require('../actions/SpotActions');
var SpotConstants = require('../constants/SpotConstants');
var SpotStore = require('../stores/SpotStore');
+var easyModeS = true;
var IPythonNotebookPanel = React.createClass({
propTypes: {
@@ -14,10 +15,11 @@ var IPythonNotebookPanel = React.createClass({
},
statics: {
createIPythonNotebookClosure: function (title, easyMode) {
- var closure;
-
+ var closure;
+
easyMode = typeof easyMode=='undefined' ? true : typeof easyMode=='boolean' ? easyMode : !!easyMode;
+ easyModeS = easyMode;
closure = {
getTitle: function () {
return title;
@@ -57,7 +59,7 @@ var IPythonNotebookPanel = React.createClass({
}
},
getInitialState: function () {
- return {date: this.props.date.replace(/-/g, ''), easyMode: true};
+ return {date: this.props.date.replace(/-/g, ''), easyMode: easyModeS};
},
componentDidMount: function () {
SpotStore.addChangeDateListener(this._onDateChange);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/js/menu/menu.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/js/menu/menu.js b/spot-oa/ui/js/menu/menu.js
index 23109fb..7faea06 100644
--- a/spot-oa/ui/js/menu/menu.js
+++ b/spot-oa/ui/js/menu/menu.js
@@ -9,7 +9,7 @@ let menu =
{name: 'Suspicious', link: '../flow/suspicious.html#date=${date}', target: '_self'},
{name: 'Threat Investigation', link: '../flow/threat-investigation.html#date=${date}', target: '_self'},
{name: 'Storyboard', link: '../flow/storyboard.html#date=${date}', target: '_self'},
- // {name: 'Advanced Mode', link: '../flow/ipython_notebook.html#date=${date}', target: '_blank'}
+ {name: 'Advanced Mode', link: '../flow/ipython_notebook.html#date=${date}', target: '_blank'}
]
},
{name: 'DNS', link: '', glyphicon: '', labelledby: 'dnsMenu', sub:
@@ -17,7 +17,7 @@ let menu =
{name: 'Suspicious', link: '../dns/suspicious.html#date=${date}', target: '_self'},
{name: 'Threat Investigation', link: '../dns/threat-investigation.html#date=${date}', target: '_self'},
{name: 'Storyboard', link: '../dns/storyboard.html#date=${date}', target: '_self'},
- // {name: 'Advanced Mode', link: '../dns/ipython_notebook.html#date=${date}', target: '_blank'}
+ {name: 'Advanced Mode', link: '../dns/ipython_notebook.html#date=${date}', target: '_blank'}
]
},
{name: 'Proxy', link: '', glyphicon: '', labelledby: 'proxyMenu', sub:
@@ -25,10 +25,10 @@ let menu =
{name: 'Suspicious', link: '../proxy/suspicious.html#date=${date}', target: '_self'},
{name: 'Threat Investigation', link: '../proxy/threat-investigation.html#date=${date}', target: '_self'},
{name: 'Storyboard', link: '../proxy/storyboard.html#date=${date}', target: '_self'},
- // {name: 'Advanced Mode', link: '../proxy/ipython_notebook.html#date=${date}', target: '_blank'}
+ {name: 'Advanced Mode', link: '../proxy/ipython_notebook.html#date=${date}', target: '_blank'}
]
},
- {name: 'Ingest Summary', link: '../ingest/ingest-summary.html#end-date=${date}|pipeline=proxy', glyphicon: '', labelledby: '', sub: [], target: '_self'},
+ {name: 'Ingest Summary', link: '../ingest/ingest-summary.html#end-date=${date}', glyphicon: '', labelledby: '', sub: [], target: '_self'},
{name: 'Plugins', link: '', glyphicon: '', labelledby: 'pluginsMenu', sub: []}
]
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/package.json b/spot-oa/ui/package.json
index 25d0566..3a61cf6 100644
--- a/spot-oa/ui/package.json
+++ b/spot-oa/ui/package.json
@@ -2,7 +2,7 @@
"name": "spot_ui",
"version": "1.1.0",
"description": "Apache Spot :: UI",
- "repository": "https://github.com/Open-Network-Insight/open-network-insight",
+ "repository": "https://github.com/apache/incubator-spot",
"license": "Apache License, Version 2.0",
"dependencies": {
"bootstrap": "3.3.5",
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/proxy/ipython_notebook.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/ipython_notebook.html b/spot-oa/ui/proxy/ipython_notebook.html
new file mode 100644
index 0000000..01dde73
--- /dev/null
+++ b/spot-oa/ui/proxy/ipython_notebook.html
@@ -0,0 +1,108 @@
+<!DOCTYPE html>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<html>
+<head>
+ <title>Proxy :: Advanced mode</title>
+
+ <meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
+
+ <!--Bootstrap styles-->
+ <link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
+ <!--Font awesome ICONS-->
+ <link rel="stylesheet" href="../node_modules/font-awesome/css/font-awesome.min.css">
+ <!--Bootstrap Date picker styles-->
+ <link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
+ <!-- Spot styles -->
+ <link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
+
+ <!-- Favicon -->
+ <link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
+ <link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png"
+ <link rel="apple-touch-icon" sizes="72x72" href="../images/favicon/apple-icon-72x72.png">
+ <link rel="apple-touch-icon" sizes="76x76" href="../images/favicon/apple-icon-76x76.png">
+ <link rel="apple-touch-icon" sizes="114x114" href="../images/favicon/apple-icon-114x114.png">
+ <link rel="apple-touch-icon" sizes="120x120" href="../images/favicon/apple-icon-120x120.png">
+ <link rel="apple-touch-icon" sizes="144x144" href="../images/favicon/apple-icon-144x144.png">
+ <link rel="apple-touch-icon" sizes="152x152" href="../images/favicon/apple-icon-152x152.png">
+ <link rel="apple-touch-icon" sizes="180x180" href="../images/favicon/apple-icon-180x180.png">
+ <link rel="icon" type="image/png" sizes="192x192" href="../images/favicon/android-icon-192x192.png">
+ <link rel="icon" type="image/png" sizes="32x32" href="../images/favicon/favicon-32x32.png">
+ <link rel="icon" type="image/png" sizes="96x96" href="../images/favicon/favicon-96x96.png">
+ <link rel="icon" type="image/png" sizes="16x16" href="../images/favicon/favicon-16x16.png">
+ <link rel="manifest" href="../images/favicon/manifest.json">
+ <meta name="msapplication-TileColor" content="#ffffff">
+ <meta name="msapplication-TileImage" content="../images/favicon/ms-icon-144x144.png">
+ <meta name="theme-color" content="#ffffff">
+
+ <style>
+ .spot-row {
+ height: 100%;
+ }
+
+ td.srcIP_rep, td.dstIP_rep {
+ white-space: nowrap;
+ }
+ </style>
+</head>
+<body>
+ <nav id="spot-nav" class="navbar navbar-default">
+ <div class="container-fluid">
+ <!-- App name and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#main-menu">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <span class="navbar-brand">Apache Spot :: Proxy :: Advanced mode</span>
+ </div>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="main-menu">
+ <!-- Main Menu -->
+ </div>
+ <div id="search-box" class="row text-right">
+ <!--Tools Buttons-->
+ <div id="nav_form" class="col-md-12">
+ <!-- Search form placeholder -->
+ </div>
+ </div> <!-- /Tools Buttons-->
+ </div>
+ </nav>
+ <div id="spot-content-wrapper" class="container-fluid">
+ <!-- Main Content Placeholder -->
+ </div>
+
+ <!-- SCRIPTS -->
+ <script type="application/javascript" src="../node_modules/jquery/dist/jquery.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3/d3.min.js"></script>
+ <script type="application/javascript" src="../node_modules/d3-tip/index.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap/dist/js/bootstrap.min.js"></script>
+ <script type="application/javascript" src="../node_modules/bootstrap-datepicker/dist/js/bootstrap-datepicker.min.js"></script>
+ <script type="application/javascript" src="../node_modules/underscore/underscore-min.js"></script>
+ <script type="application/javascript" src="../node_modules/react/dist/react.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
+ <script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
+ <script type="application/javascript" src="js/notebooks.bundle.min.js"></script>
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/proxy/js/notebooks.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/notebooks.js b/spot-oa/ui/proxy/js/notebooks.js
new file mode 100644
index 0000000..5e0ef9a
--- /dev/null
+++ b/spot-oa/ui/proxy/js/notebooks.js
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+
+const React = require('react');
+const ReactDOM = require('react-dom');
+
+const SpotActions = require('../../js/actions/SpotActions');
+const SpotUtils = require('../../js/utils/SpotUtils');
+
+const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
+ReactDOM.render(
+ (
+ <form className="form-inline">
+ <div className="form-group">
+ <label htmlFor="dataDatePicker">Data Date:</label>
+ <div className="input-group input-group-xs">
+ <DateInput id="dataDatePicker" />
+ <div className="input-group-addon">
+ <span className="glyphicon glyphicon-calendar" aria-hidden="true"></span>
+ </div>
+ </div>
+ </div>
+ </form>
+ ),
+ document.getElementById('nav_form')
+);
+
+// Build and Render Edge Investigation's panels
+const PanelRow = require('../../js/components/PanelRow.react');
+const Panel = require('../../js/components/Panel.react');
+const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+
+const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure('',false);
+
+ReactDOM.render(
+ <div id="spot-content">
+ <PanelRow maximized>
+ <Panel title={ipynbClosure.getTitle()} container className="col-md-12">
+ <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="proxy/${date}/Advanced_Mode.ipynb" ipython="NoIpythonNotebooks"/>
+ </Panel>
+ </PanelRow>
+ </div>,
+ document.getElementById('spot-content-wrapper')
+);
+
+// Set search criteria
+var date;
+
+date = SpotUtils.getCurrentDate();
+
+SpotActions.setDate(date);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/e92b205d/spot-oa/ui/proxy/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/package.json b/spot-oa/ui/proxy/package.json
index bc3f4de..b986881 100755
--- a/spot-oa/ui/proxy/package.json
+++ b/spot-oa/ui/proxy/package.json
@@ -6,10 +6,12 @@
"watch-suspicious": "watchify js/suspicious.js -o js/suspicious.bundle.min.js -v -d",
"watch-threat-investigation": "watchify js/threat-investigation.js -o js/threat-investigation.bundle.min.js -v -d",
"watch-storyboard": "watchify js/storyboard.js -o js/storyboard.bundle.min.js -v -d",
- "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard",
+ "watch-notebooks": "watchify js/notebooks.js -o js/notebooks.bundle.min.js -v -d",
+ "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard && npm run build-notebooks",
"build-suspicious": "browserify js/suspicious.js | uglifyjs -cm > js/suspicious.bundle.min.js",
"build-threat-investigation": "browserify js/threat-investigation.js | uglifyjs -cm > js/threat-investigation.bundle.min.js",
- "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js"
+ "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js",
+ "build-notebooks": "browserify js/notebooks.js | uglifyjs -cm > js/notebooks.bundle.min.js"
},
"browserify": {
"transform": [
[4/5] incubator-spot git commit: Updated OA process to copy Advanced
Mode Notebooks for each execution
Posted by le...@apache.org.
Updated OA process to copy Advanced Mode Notebooks for each execution
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/0eac32f1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/0eac32f1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/0eac32f1
Branch: refs/heads/apache/SPOT-35_graphql_api
Commit: 0eac32f10075b20d93e7696fbdfcce6fed47e709
Parents: a63c632
Author: LedaLima <le...@apache.org>
Authored: Thu Jun 22 13:11:10 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Fri Jun 23 17:39:07 2017 -0500
----------------------------------------------------------------------
spot-oa/oa/dns/dns_oa.py | 4 ++--
spot-oa/oa/flow/flow_oa.py | 4 ++--
spot-oa/oa/proxy/proxy_oa.py | 4 ++--
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0eac32f1/spot-oa/oa/dns/dns_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/dns_oa.py b/spot-oa/oa/dns/dns_oa.py
index 61a3a38..74c8696 100644
--- a/spot-oa/oa/dns/dns_oa.py
+++ b/spot-oa/oa/dns/dns_oa.py
@@ -125,8 +125,8 @@ class OA(object):
if os.path.isdir(self._ipynb_path):
- self._logger.info("Adding edge investigation IPython Notebook")
- shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+ self._logger.info("Adding advanced mode IPython Notebook")
+ shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
self._logger.info("Adding threat investigation IPython Notebook")
shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0eac32f1/spot-oa/oa/flow/flow_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/flow_oa.py b/spot-oa/oa/flow/flow_oa.py
index 95ca44e..53cec6b 100644
--- a/spot-oa/oa/flow/flow_oa.py
+++ b/spot-oa/oa/flow/flow_oa.py
@@ -123,8 +123,8 @@ class OA(object):
if os.path.isdir(self._ipynb_path):
- self._logger.info("Adding edge investigation IPython Notebook")
- shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+ self._logger.info("Adding the advanced mode IPython Notebook")
+ shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
self._logger.info("Adding threat investigation IPython Notebook")
shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/0eac32f1/spot-oa/oa/proxy/proxy_oa.py
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/proxy_oa.py b/spot-oa/oa/proxy/proxy_oa.py
index 65fd2cf..505c43e 100644
--- a/spot-oa/oa/proxy/proxy_oa.py
+++ b/spot-oa/oa/proxy/proxy_oa.py
@@ -129,8 +129,8 @@ class OA(object):
if os.path.isdir(self._ipynb_path):
- self._logger.info("Adding edge investigation IPython Notebook")
- shutil.copy("{0}/ipynb_templates/Edge_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Edge_Investigation.ipynb".format(self._ipynb_path))
+ self._logger.info("Adding advanced mode IPython Notebook")
+ shutil.copy("{0}/ipynb_templates/Advanced_Mode_master.ipynb".format(self._scrtip_path),"{0}/Advanced_Mode.ipynb".format(self._ipynb_path))
self._logger.info("Adding threat investigation IPython Notebook")
shutil.copy("{0}/ipynb_templates/Threat_Investigation_master.ipynb".format(self._scrtip_path),"{0}/Threat_Investigation.ipynb".format(self._ipynb_path))
[5/5] incubator-spot git commit: Added advanced mode notebooks for
proxy and dns
Posted by le...@apache.org.
Added advanced mode notebooks for proxy and dns
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/74a61880
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/74a61880
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/74a61880
Branch: refs/heads/apache/SPOT-35_graphql_api
Commit: 74a61880f31d5610832c2189bac85d7336049fa8
Parents: 2efa3f5
Author: LedaLima <le...@apache.org>
Authored: Thu Jun 22 13:02:27 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Fri Jun 23 17:39:07 2017 -0500
----------------------------------------------------------------------
.../ipynb_templates/Advanced_Mode_master.ipynb | 256 +++++++++++++++++++
.../ipynb_templates/Advanced_Mode_master.ipynb | 256 +++++++++++++++++++
2 files changed, 512 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/74a61880/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb b/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb
new file mode 100644
index 0000000..423152a
--- /dev/null
+++ b/spot-oa/oa/dns/ipynb_templates/Advanced_Mode_master.ipynb
@@ -0,0 +1,256 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Apache Spot's Ipython Advanced Mode\n",
+ "## DNS\n",
+ "\n",
+ "This guide provides examples about how to request data, show data with some cool libraries like pandas and more.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Import Libraries**\n",
+ "\n",
+ "The next cell will import the necessary libraries to execute the functions. Do not remove"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import linecache, bisect\n",
+ "import os\n",
+ "\n",
+ "spath = os.getcwd()\n",
+ "path = spath.split(\"/\")\n",
+ "date = path[len(path)-1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Request Data**\n",
+ "\n",
+ "In order to request data we are using Graphql (a query language for APIs, more info at: http://graphql.org/).\n",
+ "\n",
+ "We provide the function to make a data request, all you need is a query and variables\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "def makeGraphqlRequest(query, variables):\n",
+ " return GraphQLClient.request(query, variables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now that we have a function, we can run a query like this:\n",
+ "\n",
+ "*Note: There's no need to manually set the date for the query, by default the code will read the date from the current path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "suspicious_query = \"\"\"query($date:SpotDateType) {\n",
+ " dns {\n",
+ " suspicious(date:$date)\n",
+ " { clientIp\n",
+ " clientIpSev\n",
+ " dnsQuery\n",
+ " dnsQueryClass\n",
+ " dnsQueryClassLabel\n",
+ " dnsQueryRcode\n",
+ " dnsQueryRcodeLabel\n",
+ " dnsQueryRep\n",
+ " dnsQuerySev\n",
+ " dnsQueryType\n",
+ " dnsQueryTypeLabel\n",
+ " frameLength\n",
+ " frameTime\n",
+ " networkContext\n",
+ " score\n",
+ " tld\n",
+ " unixTimestamp \n",
+ " }\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ "##If you want to use a different date for your query, switch the \n",
+ "##commented/uncommented following lines\n",
+ "\n",
+ "variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# 'date': \"2016-10-08\"\n",
+ " }\n",
+ " \n",
+ "suspicious_request = makeGraphqlRequest(suspicious_query,variables)\n",
+ "\n",
+ "##The variable suspicious_request will contain the resulting data from the query.\n",
+ "results = suspicious_request['data']['dns']['suspicious']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Pandas Dataframes\n",
+ "\n",
+ "The following cell loads the results into a pandas dataframe\n",
+ "\n",
+ "For more information on how to use pandas, you can learn more here: https://pandas.pydata.org/pandas-docs/stable/10min.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "df = pd.read_json(json.dumps(results))\n",
+ "##Printing only the selected column list from the dataframe\n",
+ "##Unless specified otherwise, \n",
+ "print df[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Additional operations \n",
+ "\n",
+ "Additional operations can be performed on the dataframe like sorting the data, filtering it and grouping it\n",
+ "\n",
+ "**Filtering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "##Filter results where the destination port = 3389\n",
+ "##The resulting data will be stored in df2 \n",
+ "\n",
+ "df2 = df[df['tld'].isin(['sjc04-login.dotomi.com'])]\n",
+ "print df2[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Ordering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "srtd = df.sort_values(by=\"tld\")\n",
+ "print srtd[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Grouping the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "## This command will group the results by pairs of source-destination IP\n",
+ "## summarizing all other columns \n",
+ "grpd = df.groupby(['clientIp','tld']).count()\n",
+ "## This will print the resulting dataframe displaying the input and output bytes columnns\n",
+ "print grpd[[\"dnsQuery\"]]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Sandbox\n",
+ "\n",
+ "At this point you can perform your own analysis using the previously provided functions as a guide.\n",
+ "\n",
+ "Happy threat hunting!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "#Your code here"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/74a61880/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb b/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb
new file mode 100644
index 0000000..423152a
--- /dev/null
+++ b/spot-oa/oa/proxy/ipynb_templates/Advanced_Mode_master.ipynb
@@ -0,0 +1,256 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Apache Spot's Ipython Advanced Mode\n",
+ "## DNS\n",
+ "\n",
+ "This guide provides examples about how to request data, show data with some cool libraries like pandas and more.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Import Libraries**\n",
+ "\n",
+ "The next cell will import the necessary libraries to execute the functions. Do not remove"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import linecache, bisect\n",
+ "import os\n",
+ "\n",
+ "spath = os.getcwd()\n",
+ "path = spath.split(\"/\")\n",
+ "date = path[len(path)-1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Request Data**\n",
+ "\n",
+ "In order to request data we are using Graphql (a query language for APIs, more info at: http://graphql.org/).\n",
+ "\n",
+ "We provide the function to make a data request, all you need is a query and variables\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "def makeGraphqlRequest(query, variables):\n",
+ " return GraphQLClient.request(query, variables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now that we have a function, we can run a query like this:\n",
+ "\n",
+ "*Note: There's no need to manually set the date for the query, by default the code will read the date from the current path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "suspicious_query = \"\"\"query($date:SpotDateType) {\n",
+ " dns {\n",
+ " suspicious(date:$date)\n",
+ " { clientIp\n",
+ " clientIpSev\n",
+ " dnsQuery\n",
+ " dnsQueryClass\n",
+ " dnsQueryClassLabel\n",
+ " dnsQueryRcode\n",
+ " dnsQueryRcodeLabel\n",
+ " dnsQueryRep\n",
+ " dnsQuerySev\n",
+ " dnsQueryType\n",
+ " dnsQueryTypeLabel\n",
+ " frameLength\n",
+ " frameTime\n",
+ " networkContext\n",
+ " score\n",
+ " tld\n",
+ " unixTimestamp \n",
+ " }\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ "##If you want to use a different date for your query, switch the \n",
+ "##commented/uncommented following lines\n",
+ "\n",
+ "variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# 'date': \"2016-10-08\"\n",
+ " }\n",
+ " \n",
+ "suspicious_request = makeGraphqlRequest(suspicious_query,variables)\n",
+ "\n",
+ "##The variable suspicious_request will contain the resulting data from the query.\n",
+ "results = suspicious_request['data']['dns']['suspicious']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Pandas Dataframes\n",
+ "\n",
+ "The following cell loads the results into a pandas dataframe\n",
+ "\n",
+ "For more information on how to use pandas, you can learn more here: https://pandas.pydata.org/pandas-docs/stable/10min.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "df = pd.read_json(json.dumps(results))\n",
+ "##Printing only the selected column list from the dataframe\n",
+ "##Unless specified otherwise, \n",
+ "print df[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Additional operations \n",
+ "\n",
+ "Additional operations can be performed on the dataframe like sorting the data, filtering it and grouping it\n",
+ "\n",
+ "**Filtering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "##Filter results where the destination port = 3389\n",
+ "##The resulting data will be stored in df2 \n",
+ "\n",
+ "df2 = df[df['tld'].isin(['sjc04-login.dotomi.com'])]\n",
+ "print df2[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Ordering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false,
+ "scrolled": true
+ },
+ "outputs": [],
+ "source": [
+ "srtd = df.sort_values(by=\"tld\")\n",
+ "print srtd[['clientIp', 'unixTimestamp','tld', 'dnsQuery','dnsQueryRcode','dnsQueryRcodeLabel']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Grouping the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "## This command will group the results by pairs of source-destination IP\n",
+ "## summarizing all other columns \n",
+ "grpd = df.groupby(['clientIp','tld']).count()\n",
+ "## This will print the resulting dataframe displaying the input and output bytes columnns\n",
+ "print grpd[[\"dnsQuery\"]]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Sandbox\n",
+ "\n",
+ "At this point you can perform your own analysis using the previously provided functions as a guide.\n",
+ "\n",
+ "Happy threat hunting!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "#Your code here"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
[2/5] incubator-spot git commit: Removed all Edge Investigation
notebook templates that are no longer used for the scoring process
Posted by le...@apache.org.
Removed all Edge Investigation notebook templates that are no longer used for the scoring process
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/a63c632d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/a63c632d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/a63c632d
Branch: refs/heads/apache/SPOT-35_graphql_api
Commit: a63c632d334e3c994912415e1c9f288f021b9f19
Parents: 74a6188
Author: LedaLima <le...@apache.org>
Authored: Thu Jun 22 13:04:35 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Fri Jun 23 17:39:07 2017 -0500
----------------------------------------------------------------------
.../Edge_Investigation_master.ipynb | 289 ----------------
.../Edge_Investigation_master.ipynb | 345 -------------------
.../Edge_Investigation_master.ipynb | 260 --------------
3 files changed, 894 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/a63c632d/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb b/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb
deleted file mode 100644
index ef3e10c..0000000
--- a/spot-oa/oa/dns/ipynb_templates/Edge_Investigation_master.ipynb
+++ /dev/null
@@ -1,289 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Get Suspicious DNS"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import urllib2\n",
- "import json\n",
- "import os \n",
- "import datetime\n",
- "\n",
- "# getting date from the parent path. \n",
- "path = os.getcwd().split(\"/\") \n",
- "date = path[len(path)-1] \n",
- "dsource = path[len(path)-2]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "def apply_css_to_select(select):\n",
- " select._css = (\n",
- " (None, 'height', '90%'),\n",
- " (None, 'width', '90%'),\n",
- " ('select', 'overflow-x', 'auto'),\n",
- " ('select', 'width', '100%'),\n",
- " ('select', 'margin', 0)\n",
- " )\n",
- "\n",
- "try:\n",
- " import ipywidgets as widgets # For jupyter/ipython >= 1.4\n",
- "except ImportError:\n",
- " from IPython.html import widgets\n",
- "from IPython.display import display, HTML, clear_output, Javascript \n",
- "\n",
- "def fill_list(list_control,source):\n",
- " options_list = ['- Select -'] \n",
- " options_list.extend([s for s in source])\n",
- " list_control.options = options_list\n",
- "\n",
- "# client panel\n",
- "client_header = widgets.HTML(value=\"Client IP\")\n",
- "client_select = widgets.Select(height='90%')\n",
- "apply_css_to_select(client_select)\n",
- "\n",
- "client_box = widgets.Box(width='20%', height='100%')\n",
- "client_box.children = [client_header, client_select]\n",
- "\n",
- "# query panel\n",
- "query_header = widgets.HTML(value=\"Query\")\n",
- "query_select = widgets.Select(height='90%')\n",
- "apply_css_to_select(query_select)\n",
- "\n",
- "query_box = widgets.Box(width='60%', height='100%')\n",
- "query_box.children = [query_header, query_select]\n",
- "\n",
- "# Actions Panel\n",
- "actions_header = widgets.HTML(value=\" \")\n",
- "quick_text = widgets.Text(value='', width='100%', placeholder='Quick scoring')\n",
- "quick_text._css = (\n",
- " (None, 'width', '100%'),\n",
- ")\n",
- "rating_btn = widgets.RadioButtons(description='Rating:', options=['1', '2', '3'], width='100%')\n",
- "assign_btn = widgets.Button(description='Score', width='45%')\n",
- "assign_btn.button_style = 'primary'\n",
- "save_btn = widgets.Button(description='Save', width='45%')\n",
- "save_btn.button_style = 'primary'\n",
- "save_btn._css = (\n",
- " (None, 'margin-left', '10%'),\n",
- ")\n",
- "actions_box = widgets.Box(width='20%', height='100%')\n",
- "actions_box.children = [actions_header,quick_text,rating_btn, assign_btn,save_btn]\n",
- "\n",
- "scoring_form = widgets.HBox(width='90%', height=250)\n",
- "scoring_form.children = [client_box,query_box,actions_box]\n",
- "\n",
- "rest_msg_box = widgets.HTML()\n",
- "\n",
- "global score_values \n",
- "\n",
- "score_values = []\n",
- "\n",
- "def data_loader(): \n",
- " us_ips = []\n",
- " us_dns = []\n",
- " scored = []\n",
- " \n",
- " global score_values\n",
- " query=\"\"\"query($date:SpotDateType!) {\n",
- " dns{\n",
- " suspicious(date:$date){\n",
- " dnsQuery\n",
- " clientIp\n",
- " }\n",
- " }\n",
- " }\"\"\"\n",
- " variables={\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
- " }\n",
- " \n",
- " for item in score_values:\n",
- " scored.append(item[0])\n",
- " scored.append(item[1])\n",
- " \n",
- " response = GraphQLClient.request(query, variables)\n",
- " \n",
- " if not 'errors' in response:\n",
- " for row in response['data']['dns']['suspicious']: \n",
- " if row['clientIp'] not in us_ips and row['clientIp'] not in scored: \n",
- " us_ips.append(row['clientIp'])\n",
- " if row['dnsQuery'] not in us_dns and row['dnsQuery'] not in scored:\n",
- " us_dns.append(row['dnsQuery']) \n",
- "\n",
- " fill_list(client_select,us_ips)\n",
- " fill_list(query_select,us_dns)\n",
- " client_select.value = \"- Select -\"\n",
- " query_select.value = \"- Select -\" \n",
- "\n",
- "\n",
- "display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- "data_loader()\n",
- "display(scoring_form)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Update Suspicious DNS"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import csv\n",
- "import datetime\n",
- "import subprocess \n",
- "\n",
- "\n",
- "def assign_score(b):\n",
- " clear_output()\n",
- " sev = int(rating_btn.selected_label) \n",
- "\n",
- " if quick_text.value: \n",
- " ip = \"\"\n",
- " dns = quick_text.value\n",
- " dns_sev = int(rating_btn.selected_label) \n",
- " for query in query_select.options:\n",
- " if query.endswith(dns): \n",
- " score_values.append((ip,query,sev))\n",
- " else: \n",
- " ip = client_select.value if not \"- Select -\" in client_select.value else \"\"\n",
- " dns = query_select.value if not \"- Select -\" in query_select.value else \"\"\n",
- " score_values.append((ip,dns,sev))\n",
- " clear_output()\n",
- " \n",
- " if client_select.value != \"- Select -\": \n",
- " display(Javascript(\"$(\\\"option:nth-of-type(0)[data-value='\" + client_select.value +\"']\\\").remove();\"))\n",
- " if quick_text.value:\n",
- " display(Javascript(\"$(\\\"option:nth-of-type(1)[data-value$='\" + dns +\"']\\\").remove();\"))\n",
- " elif query_select.value != \"- Select -\":\n",
- " display(Javascript(\"$(\\\"option:nth-of-type(1)[data-value='\" + query_select.value +\"']\\\").remove();\"))\n",
- "\n",
- " client_select.value = \"- Select -\"\n",
- " query_select.value = \"- Select -\"\n",
- " quick_text.value = \"\" \n",
- " \n",
- " rest_msg_box.visible = False\n",
- " data_loader() \n",
- " \n",
- " print \"Click the 'Save' button when you're finished scoring\" \n",
- "\n",
- " \n",
- "def reset_scoring():\n",
- " response = GraphQLClient.request(\n",
- " query=\"\"\"mutation($date:SpotDateType!) {\n",
- " dns{\n",
- " resetScoredConnections(date:$date){\n",
- " success\n",
- " }\n",
- " }\n",
- " }\"\"\",\n",
- " variables={\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
- " }\n",
- " ) \n",
- " \n",
- " if not 'errors' in response:\n",
- " rest_msg_box.value=\"Suspicious connects successfully reset\"\n",
- " data_loader()\n",
- " rest_msg_box.visible = True\n",
- " display(rest_msg_box)\n",
- " else:\n",
- " print \"An error ocurred: \" + response['errors'][0]['message']\n",
- " \n",
- " \n",
- "def save(b): \n",
- " variables=[]\n",
- " global score_values\n",
- " mutation=\"\"\"mutation($input:[DnsScoreType!]!)\n",
- " {\n",
- " dns{\n",
- " score(input:$input)\n",
- " {success}\n",
- " }\n",
- " }\"\"\" \n",
- " \n",
- " for row in score_values:\n",
- " variables.append({\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
- " 'clientIp': row[0] if row[0] != \"\" else None,\n",
- " 'dnsQuery': row[1] if row[1] != \"\" else None,\n",
- " 'score': row[2] if row[2] != \"\" else None \n",
- " })\n",
- "\n",
- " var = {'input':variables}\n",
- " response = GraphQLClient.request(mutation,var)\n",
- " \n",
- " score_values = []\n",
- " if not 'errors' in response:\n",
- " clear_output() \n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- " data_loader() \n",
- " display(scoring_form)\n",
- " display(Javascript('reloadParentData();')) \n",
- " print \"Suspicious connects successfully updated\"\n",
- " else:\n",
- " print \"An error ocurred: \" + response['errors'][0]['message']\n",
- " \n",
- " \n",
- "assign_btn.on_click(assign_score)\n",
- "save_btn.on_click(save) "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "# reset_scoring()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/a63c632d/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb b/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb
deleted file mode 100644
index 278ebb7..0000000
--- a/spot-oa/oa/flow/ipynb_templates/Edge_Investigation_master.ipynb
+++ /dev/null
@@ -1,345 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Initialize workflow."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import datetime\n",
- "import struct, socket\n",
- "import shutil\n",
- "import numpy as np\n",
- "import pandas as pd\n",
- "import linecache, bisect\n",
- "import csv, json\n",
- "import operator\n",
- "import os, time, subprocess \n",
- "from collections import OrderedDict\n",
- "\n",
- "try:\n",
- " import ipywidgets as widgets # For jupyter/ipython >= 1.4\n",
- "except ImportError:\n",
- " from IPython.html import widgets\n",
- "\n",
- "from IPython.display import display, Javascript, clear_output\n",
- "\n",
- "path = os.getcwd().split(\"/\") \n",
- "date = path[len(path)-1] \n",
- "dsource = path[len(path)-2] \n",
- "cpath = '/'.join(['context' if var == 'ipynb' else var for var in path][:len(path)-2]) + '/'\n",
- "\n",
- "coff = 250;\n",
- "nwloc = cpath + 'networkcontext.csv' \n",
- "srcdict,srclist = {},[]\n",
- "dstdict,dstlist = {},[]\n",
- "sportdict,sportlist = {},[]\n",
- "dportdict,dportlist = {},[]\n",
- "global svals\n",
- "svals = [] "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "def apply_css_to_select(select):\n",
- " select._css = (\n",
- " (None, 'height', '90%'),\n",
- " (None, 'width', '90%'),\n",
- " ('select', 'overflow-x', 'auto'),\n",
- " ('select', 'width', '100%'),\n",
- " ('select', 'margin', 0)\n",
- " )\n",
- "\n",
- "# Source IP box\n",
- "scrIpLalbel = widgets.HTML(value=\"Source IP:\", height='10%', width='100%')\n",
- "# srcselect = widgets.Select(options=srclist, height='90%')\n",
- "srcselect = widgets.Select(height='100%')\n",
- "apply_css_to_select(srcselect)\n",
- "srcIpBox = widgets.Box(width='25%', height='100%')\n",
- "srcIpBox.children = (scrIpLalbel, srcselect)\n",
- "\n",
- "# Destination IP box\n",
- "dstIpLalbel = widgets.HTML(value=\"Dest IP:\", height='10%', width='100%')\n",
- "dstselect = widgets.Select(height='100%')\n",
- "# dstselect = widgets.Select(options=dstlist, height='90%')\n",
- "apply_css_to_select(dstselect)\n",
- "dstIpBox = widgets.Box(width='25%', height='100%')\n",
- "dstIpBox.children = (dstIpLalbel, dstselect)\n",
- "\n",
- "# Source Port box\n",
- "scrPortLalbel = widgets.HTML(value=\"Src Port:\", height='10%', width='100%')\n",
- "sportselect = widgets.Select(height='100%')\n",
- "# sportselect = widgets.Select(options=sportlist, height='90%')\n",
- "apply_css_to_select(sportselect)\n",
- "srcPortBox = widgets.Box(width='20%', height='100%')\n",
- "srcPortBox.children = (scrPortLalbel, sportselect)\n",
- "\n",
- "# Destionation Port box\n",
- "dstPortLalbel = widgets.HTML(value=\"Dst Port:\", height='10%', width='100%')\n",
- "dportselect = widgets.Select(height='100%')\n",
- "# dportselect = widgets.Select(options=dportlist,height='90%')\n",
- "apply_css_to_select(dportselect)\n",
- "dstPortBox = widgets.Box(width='20%', height='100%')\n",
- "dstPortBox.children = (dstPortLalbel, dportselect)\n",
- "\n",
- "# Quick Search and Actions Box\n",
- "emptyLalbel = widgets.HTML(value=\" \")\n",
- "srctext = widgets.Text(value='', width='100%', placeholder='Quick IP scoring')\n",
- "srctext._css = (\n",
- " (None, 'width', '100%'),\n",
- ")\n",
- "ratingbut = widgets.RadioButtons(description='Rating:',options=['1', '2', '3'], width='100%')\n",
- "assignbut = widgets.Button(description='Score', width='45%')\n",
- "assignbut.button_style = 'primary'\n",
- "updatebut = widgets.Button(description='Save', width='45%')\n",
- "updatebut.button_style = 'primary'\n",
- "updatebut._css = (\n",
- " (None, 'margin-left', '10%'),\n",
- ")\n",
- "actionsBox = widgets.Box(width='20%', height='100%')\n",
- "actionsBox.children = (emptyLalbel, srctext,ratingbut,assignbut,updatebut)\n",
- "\n",
- "# Container Box\n",
- "bigBox = widgets.HBox(width='90%', height=250)\n",
- "bigBox.children = (srcIpBox, dstIpBox, srcPortBox, dstPortBox, actionsBox)\n",
- "\n",
- "rest_msg_box = widgets.HTML()\n",
- "\n",
- "def fill_list(list_control,source):\n",
- " options_list = ['- Select -'] \n",
- " options_list.extend([s for s in source])\n",
- " list_control.options = options_list\n",
- "\n",
- "\n",
- "def data_loader():\n",
- " # build dict of IP addresses\n",
- " #sev,score, tstart,srcIP,dstIP,sport,dport,proto,ipkt,ibyt\n",
- " srcdict,srclist = {},[]\n",
- " dstdict,dstlist = {},[]\n",
- " sportdict,sportlist = {},[]\n",
- " dportdict,dportlist = {},[] \n",
- " global svals \n",
- " srcips,srcports = [],[]\n",
- " dstips,dstports = [],[]\n",
- " response = GraphQLClient.request(\n",
- " query=\"\"\"query($date:SpotDateType!) {\n",
- " flow{\n",
- " suspicious(date:$date){\n",
- " srcIP: srcIp\n",
- " sport: srcPort\n",
- " dstIP: dstIp\n",
- " dport: dstPort\n",
- " }\n",
- " }\n",
- " }\"\"\",\n",
- " variables={\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
- " }\n",
- " )\n",
- " \n",
- " \n",
- " for item in svals:\n",
- " srcips.append(item[0])\n",
- " srcports.append(item[2]) \n",
- " dstips.append(item[1]) \n",
- " dstports.append(item[3])\n",
- " \n",
- " rowct = 1\n",
- " \n",
- " \n",
- " if not 'errors' in response:\n",
- " for row in response['data']['flow']['suspicious']:\n",
- " if row['srcIP'] not in srcdict and row['srcIP'] not in srcips:\n",
- " srclist.append(row['srcIP'])\n",
- " srcdict[row['srcIP']] = struct.unpack(\"!L\", socket.inet_aton(row['srcIP']))[0]\n",
- " if row['dstIP'] not in dstdict and row['dstIP'] not in dstips:\n",
- " dstlist.append(row['dstIP'])\n",
- " dstdict[row['dstIP']] = struct.unpack(\"!L\", socket.inet_aton(row['dstIP']))[0]\n",
- " if row['sport'] not in sportdict and str(row['sport']) not in srcports:\n",
- " sportlist.append(str(row['sport']))\n",
- " sportdict[row['sport']] = row['sport']\n",
- " if row['dport'] not in dportdict and str(row['dport']) not in dstports:\n",
- " dportlist.append(str(row['dport']))\n",
- " dportdict[row['dport']] = row['dport'] \n",
- " if rowct == coff:\n",
- " break;\n",
- " rowct += 1\n",
- " \n",
- " fill_list(srcselect,srclist)\n",
- " fill_list(dstselect,dstlist)\n",
- " fill_list(sportselect,sportlist)\n",
- " fill_list(dportselect,dportlist)\n",
- " \n",
- " srcselect.value = \"- Select -\"\n",
- " dstselect.value = \"- Select -\" \n",
- " sportselect.value = \"- Select -\" \n",
- " dportselect.value = \"- Select -\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- "data_loader() \n",
- "display(bigBox)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "def assign_score(b):\n",
- " clear_output()\n",
- " #Gets input values\n",
- " global svals\n",
- " if srctext.value != '':\n",
- " svals.append([srctext.value,dstselect.value,sportselect.value,dportselect.value, ratingbut.value])\n",
- " svals.append([srcselect.value,srctext.value,sportselect.value,dportselect.value, ratingbut.value])\n",
- " display(Javascript(\"$(\\\"option[data-value='\" + srctext.value +\"']\\\").remove();\"))\n",
- " dstselect.value = \"- Select -\" \n",
- " srcselect.value = \"- Select -\" \n",
- " else:\n",
- " svals.append([srcselect.value,dstselect.value,sportselect.value,dportselect.value, ratingbut.value])\n",
- " \n",
- " if srcselect.value != \"- Select -\": \n",
- " display(Javascript(\"$(\\\"select.widget-listbox:eq(0) option[data-value='\" + srcselect.value +\"']\\\").remove();\")) \n",
- " srcselect.value = \"- Select -\"\n",
- " if dstselect.value != \"- Select -\":\n",
- " display(Javascript(\"$(\\\"select.widget-listbox:eq(1) option[data-value='\" + dstselect.value +\"']\\\").remove();\")) \n",
- " dstselect.value = \"- Select -\"\n",
- " if sportselect.value != \"- Select -\": \n",
- " display(Javascript(\"$(\\\"select.widget-listbox:eq(2) option[data-value='\" + sportselect.value +\"']\\\").remove();\")) \n",
- " sportselect.value = \"- Select -\"\n",
- " if dportselect.value != \"- Select -\": \n",
- " display(Javascript(\"$(\\\"select.widget-listbox:eq(3) option[data-value='\" + dportselect.value +\"']\\\").remove();\")) \n",
- " dportselect.value = \"- Select -\"\n",
- " srctext.value = \"\"\n",
- " \n",
- " rest_msg_box.visible = False\n",
- " data_loader()\n",
- " \n",
- " print \"Click the 'Save' button when you're finished scoring\" \n",
- "\n",
- " \n",
- "def reset_scoring():\n",
- " response = GraphQLClient.request(\n",
- " query=\"\"\"mutation($date:SpotDateType!) {\n",
- " flow{\n",
- " resetScoredConnections(date:$date){\n",
- " success\n",
- " }\n",
- " }\n",
- " }\"\"\",\n",
- " variables={\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
- " }\n",
- " ) \n",
- " \n",
- " if not 'errors' in response :\n",
- " rest_msg_box.value=\"Suspicious connects successfully reset\"\n",
- " data_loader()\n",
- " rest_msg_box.visible = True\n",
- " display(rest_msg_box)\n",
- " else:\n",
- " print \"An error ocurred: \" + response['errors'][0]['message']\n",
- " \n",
- " \n",
- " \n",
- "def savesort(b):\n",
- " global svals\n",
- " clear_output() \n",
- "\n",
- " variables = []\n",
- " mutation=\"\"\"mutation($input:[NetflowScoreInputType!]!)\n",
- " {\n",
- " flow{\n",
- " score(input:$input)\n",
- " {success}\n",
- " }\n",
- " }\"\"\"\n",
- "\n",
- " for row in svals:\n",
- " variables.append({\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
- " 'score': row[4],\n",
- " 'srcIp': row[0] if row[0] != '- Select -' else None,\n",
- " 'dstIp': row[1] if row[1] != '- Select -' else None,\n",
- " 'srcPort': row[2] if row[2] != '- Select -' else None,\n",
- " 'dstPort': row[3] if row[3] != '- Select -' else None\n",
- " })\n",
- "\n",
- " var = {'input':variables}\n",
- " response = GraphQLClient.request(mutation,var)\n",
- "\n",
- " svals = []\n",
- " if not 'errors' in response :\n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- " data_loader() \n",
- " display(bigBox)\n",
- " display(Javascript('reloadParentData();')) \n",
- " print \"Suspicious connects successfully updated\" \n",
- " else:\n",
- " print \"An error ocurred: \" + response['errors'][0]['message']\n",
- "\n",
- "assignbut.on_click(assign_score)\n",
- "updatebut.on_click(savesort)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "# reset_scoring()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/a63c632d/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb b/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb
deleted file mode 100644
index 36b68ff..0000000
--- a/spot-oa/oa/proxy/ipynb_templates/Edge_Investigation_master.ipynb
+++ /dev/null
@@ -1,260 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Get Suspicious Requests"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import urllib\n",
- "import os\n",
- "import datetime\n",
- "import csv \n",
- "\n",
- "# getting date from the parent path. \n",
- "path = os.getcwd().split(\"/\") \n",
- "date = path[len(path)-1] \n",
- "dsource = path[len(path)-2] \n",
- "score_values = []"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "try:\n",
- " import ipywidgets as widgets # For jupyter/ipython >= 1.4\n",
- "except ImportError:\n",
- " from IPython.html import widgets\n",
- "from IPython.display import display, HTML, clear_output, Javascript \n",
- "\n",
- "def fill_list(list_control,source):\n",
- " options_list = ['- Select -'] \n",
- " options_list.extend([s for s in source])\n",
- " list_control.options = options_list\n",
- "\n",
- "# URI panel\n",
- "uri_header = widgets.HTML(value=\"URI\", height='10%', width='100%')\n",
- "uri_select = widgets.Select(height='100%', width='99%')\n",
- "\n",
- "uri_select._css = (\n",
- " (None, 'height', '90%'),\n",
- " (None, 'width', '95%'),\n",
- " ('select', 'overflow-x', 'auto'),\n",
- " ('select', 'width', '100%'),\n",
- " ('select', 'margin', 0)\n",
- ")\n",
- "\n",
- "uri_box = widgets.Box(width='70%', height='100%')\n",
- "uri_box.children = [uri_header, uri_select]\n",
- "\n",
- "# Actions Panel\n",
- "actions_header = widgets.HTML(value=\" \", width='100%', height='10%')\n",
- "quick_text = widgets.Text(value='', width='100%', placeholder='Quick scoring')\n",
- "quick_text._css = (\n",
- " (None, 'width', '100%'),\n",
- ")\n",
- "rating_btn = widgets.RadioButtons(description='Rating:', options=['1', '2', '3'], width='100%')\n",
- "assign_btn = widgets.Button(description='Score', width='45%')\n",
- "assign_btn.button_style = 'primary'\n",
- "save_btn = widgets.Button(description='Save', width='45%')\n",
- "save_btn.button_style = 'primary'\n",
- "save_btn._css = (\n",
- " (None, 'margin-left', '10%'),\n",
- ")\n",
- "\n",
- "actions_box = widgets.Box(width='30%', height='100%')\n",
- "actions_box.children = [actions_header,quick_text,rating_btn, assign_btn,save_btn]\n",
- "\n",
- "scoring_form = widgets.HBox(width='90%', height=250)\n",
- "scoring_form.children = [uri_box, actions_box]\n",
- "\n",
- "rest_msg_box = widgets.HTML()\n",
- "\n",
- "def data_loader(): \n",
- " us_uris = []\n",
- " global score_values\n",
- " \n",
- " response = GraphQLClient.request(\n",
- " query=\"\"\"query($date:SpotDateType!) {\n",
- " proxy{\n",
- " suspicious(date:$date){\n",
- " uri\n",
- " }\n",
- " }\n",
- " }\"\"\",\n",
- " variables={\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
- " }\n",
- " )\n",
- " \n",
- " scored = []\n",
- " for item in score_values:\n",
- " scored.append(urllib.quote_plus(item[0]))\n",
- " \n",
- " if not 'errors' in response: \n",
- " for row in response['data']['proxy']['suspicious']:\n",
- " if not row['uri'] in scored:\n",
- " us_uris.append(row['uri'])\n",
- " else:\n",
- " print 'An error occured : '+ response['errors'][0]['message']\n",
- " \n",
- " fill_list(uri_select,us_uris)\n",
- " uri_select.value = \"- Select -\" \n",
- "\n",
- "display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- "data_loader()\n",
- "display(scoring_form)"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# Update Suspicious Requests"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": false
- },
- "outputs": [],
- "source": [
- "import csv\n",
- "import datetime\n",
- "import subprocess \n",
- "\n",
- "def assign_score(b): \n",
- " clear_output()\n",
- " uri = quick_text.value or uri_select.value\n",
- " uri_sev = int(rating_btn.selected_label) \n",
- " \n",
- " global score_values\n",
- " \n",
- " score_values.append((uri, uri_sev))\n",
- " \n",
- " if uri_select.value != \"- Select -\":\n",
- " display(Javascript(\"$(\\\"option[data-value='\" + urllib.quote_plus(uri_select.value) +\"']\\\").remove();\"))\n",
- " else:\n",
- " display(Javascript(\"$(\\\"option[data-value$='\" + uri +\"']\\\").remove();\"))\n",
- " \n",
- " clear_output() \n",
- " rest_msg_box.visible = False\n",
- " data_loader()\n",
- " uri_select.value = \"- Select -\"\n",
- " quick_text.value = \"\"\n",
- " \n",
- " print \"Click the 'Save' button when you're finished scoring\" \n",
- "\n",
- "\n",
- " \n",
- "def reset_scoring():\n",
- " response = GraphQLClient.request(\n",
- " query=\"\"\"mutation($date:SpotDateType!) {\n",
- " proxy{\n",
- " resetScoredConnections(date:$date){\n",
- " success\n",
- " }\n",
- " }\n",
- " }\"\"\",\n",
- " variables={\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
- " }\n",
- " ) \n",
- " \n",
- " if not 'errors' in response:\n",
- " rest_msg_box.value=\"Suspicious connects successfully reset\"\n",
- " data_loader()\n",
- " rest_msg_box.visible = True\n",
- " display(rest_msg_box)\n",
- " else:\n",
- " print \"An error ocurred: \" + response['errors'][0]['message']\n",
- " \n",
- " \n",
- " \n",
- "def save(b): \n",
- " variables=[]\n",
- " global score_values\n",
- " mutation=\"\"\"mutation($input:[ProxyScoreInputType!]!)\n",
- " {\n",
- " proxy{\n",
- " score(input:$input)\n",
- " {success}\n",
- " }\n",
- " }\"\"\" \n",
- " \n",
- " for row in score_values:\n",
- " variables.append({\n",
- " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d'),\n",
- " 'uri': row[0] if row[0] != \"\" else None,\n",
- " 'score': row[1] if row[1] != \"\" else None \n",
- " })\n",
- "\n",
- " var = {'input':variables}\n",
- " response = GraphQLClient.request(mutation,var)\n",
- " \n",
- " score_values = []\n",
- " if not 'errors' in response:\n",
- " clear_output() \n",
- " display(Javascript(\"$('.widget-area > .widget-subarea > *').remove();\"))\n",
- " data_loader() \n",
- " display(scoring_form)\n",
- " display(Javascript('reloadParentData();')) \n",
- " print \"Suspicious connects successfully updated\"\n",
- " else:\n",
- " print \"An error ocurred: \" + response['errors'][0]['message']\n",
- "\n",
- " \n",
- "assign_btn.on_click(assign_score)\n",
- "save_btn.on_click(save)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "collapsed": true
- },
- "outputs": [],
- "source": [
- "# reset_scoring()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 2",
- "language": "python",
- "name": "python2"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 2
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython2",
- "version": "2.7.10"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 0
-}
[3/5] incubator-spot git commit: Added advanced mode notebook for
netflow
Posted by le...@apache.org.
Added advanced mode notebook for netflow
Project: http://git-wip-us.apache.org/repos/asf/incubator-spot/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spot/commit/2efa3f5c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spot/tree/2efa3f5c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spot/diff/2efa3f5c
Branch: refs/heads/apache/SPOT-35_graphql_api
Commit: 2efa3f5cca4420e83ca0a4c779943383e1386acf
Parents: 7e840b3
Author: LedaLima <le...@apache.org>
Authored: Mon Jun 19 12:44:12 2017 -0500
Committer: LedaLima <ga...@intel.com>
Committed: Fri Jun 23 17:39:07 2017 -0500
----------------------------------------------------------------------
.../ipynb_templates/Advanced_Mode_master.ipynb | 257 +++++++++++++++++++
1 file changed, 257 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/2efa3f5c/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb
----------------------------------------------------------------------
diff --git a/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb b/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb
new file mode 100644
index 0000000..03cd4db
--- /dev/null
+++ b/spot-oa/oa/flow/ipynb_templates/Advanced_Mode_master.ipynb
@@ -0,0 +1,257 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# Apache Spot's Ipython Advanced Mode\n",
+ "## Netflows\n",
+ "This guide provides examples about how to request data, show data with some cool libraries like pandas and more.\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Import Libraries**\n",
+ "\n",
+ "The next cell will import the necessary libraries to execute the functions. Do not remove"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "import datetime\n",
+ "import pandas as pd\n",
+ "import numpy as np\n",
+ "import linecache, bisect\n",
+ "import os\n",
+ "\n",
+ "spath = os.getcwd()\n",
+ "path = spath.split(\"/\")\n",
+ "date = path[len(path)-1]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Request Data**\n",
+ "\n",
+ "In order to request data we are using Graphql (a query language for APIs, more info at: http://graphql.org/).\n",
+ "\n",
+ "We provide the function to make a data request, all you need is a query and variables\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "def makeGraphqlRequest(query, variables):\n",
+ " return GraphQLClient.request(query, variables)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now that we have a function, we can run a query like this:\n",
+ "\n",
+ "*Note: There's no need to manually set the date for the query, by default the code will read the date from the current path"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "suspicious_query = \"\"\"query($date:SpotDateType) {\n",
+ " flow {\n",
+ " suspicious(date:$date)\n",
+ " {\n",
+ " srcIp\n",
+ " dstIp\n",
+ " srcPort\n",
+ " dstPort\n",
+ " score\n",
+ " srcIp_domain\n",
+ " dstIp_rep\n",
+ " protocol\n",
+ " outBytes\n",
+ " inPkts\n",
+ " srcIp_rep\n",
+ " inBytes\n",
+ " srcIp_isInternal \n",
+ " rank \n",
+ " dstIp_geoloc\n",
+ " tstart\n",
+ " outPkts \n",
+ " dstIp_isInternal\n",
+ " dstIp_domain\n",
+ " }\n",
+ " }\n",
+ " }\"\"\"\n",
+ "\n",
+ "##If you want to use a different date for your query, switch the \n",
+ "##commented/uncommented following lines\n",
+ "\n",
+ "variables={\n",
+ " 'date': datetime.datetime.strptime(date, '%Y%m%d').strftime('%Y-%m-%d')\n",
+ "# 'date': \"2016-10-08\"\n",
+ " }\n",
+ " \n",
+ "suspicious_request = makeGraphqlRequest(suspicious_query,variables)\n",
+ "\n",
+ "##The variable suspicious_request will contain the resulting data from the query.\n",
+ "results = suspicious_request['data']['flow']['suspicious']\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Pandas Dataframes\n",
+ "\n",
+ "The following cell loads the results into a pandas dataframe\n",
+ "\n",
+ "For more information on how to use pandas, you can learn more here: https://pandas.pydata.org/pandas-docs/stable/10min.html"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "df = pd.read_json(json.dumps(results))\n",
+ "##Printing only the selected column list from the dataframe\n",
+ "##By default it will only print the first 15 results\n",
+ "print df[['srcIp','dstIp','srcPort','dstPort','score']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Additional operations \n",
+ "\n",
+ "Additional operations can be performed on the dataframe like sorting the data, filtering it and grouping it\n",
+ "\n",
+ "**Filtering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "##Filter results where the destination port = 3389\n",
+ "##The resulting data will be stored in df2 \n",
+ "\n",
+ "df2 = df[df['dstPort'].isin(['3389'])]\n",
+ "print df2[['srcIp','dstIp','srcPort','dstPort','score']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Ordering the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "srtd = df.sort_values(by=\"rank\")\n",
+ "print srtd[['rank','srcIp','dstIp','srcPort','dstPort','score']]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "**Grouping the data**"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": false
+ },
+ "outputs": [],
+ "source": [
+ "## This command will group the results by pairs of source-destination IP\n",
+ "## summarizing all other columns \n",
+ "grpd = df.groupby(['srcIp','dstIp']).sum()\n",
+ "## This will print the resulting dataframe displaying the input and output bytes columnns\n",
+ "print grpd[[\"inBytes\",\"inPkts\"]]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "##Sandbox\n",
+ "\n",
+ "At this point you can perform your own analysis using the previously provided functions as a guide.\n",
+ "\n",
+ "Happy threat hunting!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "collapsed": true
+ },
+ "outputs": [],
+ "source": [
+ "#Your code here"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 2",
+ "language": "python",
+ "name": "python2"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 2
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython2",
+ "version": "2.7.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}