You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 21:37:08 UTC
[02/18] incubator-spot git commit: syncing with master
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/SuspiciousStore.js b/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
index 7c0bfe1..f79d697 100755
--- a/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
+++ b/spot-oa/ui/proxy/js/stores/SuspiciousStore.js
@@ -1,182 +1,196 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-var assign = require('object-assign');
-var d3 = require('d3');
-
-var SpotConstants = require('../../../js/constants/SpotConstants');
-var SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-var SpotUtils = require('../../../js/utils/SpotUtils');
-var ProxyConstants = require('../constants/ProxyConstants');
-var RestStore = require('../../../js/stores/RestStore');
-
-var ALL_FILTER = 'all';
-var URI_FILTER = 'fulluri';
-
-var CHANGE_FILTER_EVENT = 'change_filter';
-var HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
-var UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
-var SELECT_THREAT_EVENT = 'select_treath';
-
-var filterName = '';
-var highlightedThread = null;
-var selectedThread = null;
-
-var SuspiciousStore = assign(new RestStore(ProxyConstants.API_SUSPICIOUS), {
- _parser: d3.tsv,
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- headers: {
- p_date: 'Time',
- clientip: 'Client IP',
- host: 'Host',
- uri_rep: ' ',
- webcat: 'Web Category',
- respcode_name: 'Response Code'
- },
- ITERATOR: ['p_date', 'clientip', 'host', 'uri_rep', 'webcat', 'respcode_name'],
- getData: function () {
- var state, filter;
-
- filter = this.getFilter();
-
- if (!filter || !this._data) {
- state = this._data || {data: []};
- }
- else {
- state = assign(
- {},
- this._data
- );
-
- if (state.data) {
- state.data = state.data.filter((item) => {
- return filterName === URI_FILTER ?
- // Only look on URI field
- item[URI_FILTER].indexOf(filter) >= 0 :
- // Look on clientip and URI fields
- item.clientip == filter || item.fulluri.indexOf(filter) >= 0;
- });
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+const SpotConstants = require('../../../js/constants/SpotConstants');
+const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
+const SpotUtils = require('../../../js/utils/SpotUtils');
+
+const ObservableWithHeadersGraphQLStore = require('../../../js/stores/ObservableWithHeadersGraphQLStore');
+
+const DATE_VAR = 'date';
+const URI_VAR = 'uri';
+const CLIENT_IP_VAR = 'clientIp';
+
+const CHANGE_FILTER_EVENT = 'change_filter';
+const HIGHLIGHT_THREAT_EVENT = 'hightlight_thread';
+const UNHIGHLIGHT_THREAT_EVENT = 'unhightlight_thread';
+const SELECT_THREAT_EVENT = 'select_treath';
+
+class SuspiciousStore extends ObservableWithHeadersGraphQLStore {
+ constructor() {
+ super();
+
+ this.filterName = null;
+ this.highlightedThread = null;
+ this.selectedThread = null;
+
+ this.headers = {
+ datetime: 'Time',
+ clientip: 'Client IP',
+ host: 'Host',
+ uri_rep: ' ',
+ webcat: 'Web Category',
+ respcode_name: 'Response Code'
+ };
+
+ this.ITERATOR = ['datetime', 'clientip', 'host', 'uri_rep', 'webcat', 'respcode_name'];
+ }
+
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$uri:String,$clientIp:SpotIpType) {
+ proxy {
+ suspicious(date: $date, uri:$uri,clientIp:$clientIp) {
+ uriport: uriPort
+ uripath: uriPath
+ uriquery: uriQuery
+ datetime
+ network_context: networkContext
+ duration
+ useragent: userAgent
+ uri_rep: uriRep
+ score
+ username
+ webcat: webCategory
+ resconttype: responseContentType
+ host
+ referer
+ csbytes: clientToServerBytes
+ fulluri: uri
+ serverip: serverIp
+ reqmethod: requestMethod
+ respcode: responseCode
+ clientip: clientIp
+ respcode_name: responseCodeLabel
+ scbytes: serverToClientBytes
+ }
+ }
}
- }
+ `;
+ }
- if (state.data) {
- state.data = state.data.filter(function (item) {
- return item.uri_sev == "0";
- });
+ unboxData(data) {
+ return data.proxy.suspicious;
+ }
- if (state.data.length > SpotConstants.MAX_SUSPICIOUS_ROWS) state.data = state.data.slice(0, SpotConstants.MAX_SUSPICIOUS_ROWS);
- }
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
- return state;
- },
- setData: function (data) {
- this._data = data;
-
- this.emitChangeData();
- },
- setDate: function (date) {
- this.setEndpoint(ProxyConstants.API_SUSPICIOUS.replace('${date}', date.replace(/-/g, '')));
- },
- setFilter: function (filter) {
+ setFilter(filter) {
if (filter === '') {
- filterName = '';
- this.removeRestFilter(ALL_FILTER);
- this.removeRestFilter(URI_FILTER);
+ this.filterName = null;
+ this.unsetVariable(URI_VAR);
+ this.unsetVariable(CLIENT_IP_VAR);
}
else if (SpotUtils.IP_V4_REGEX.test(filter)) {
- this.removeRestFilter(URI_FILTER);
- this.setRestFilter(ALL_FILTER, filter);
- filterName = ALL_FILTER;
+ this.unsetVariable(URI_VAR, filter);
+ this.setVariable(CLIENT_IP_VAR, filter);
}
else {
- this.removeRestFilter(ALL_FILTER);
- this.setRestFilter(URI_FILTER, filter);
- filterName = URI_FILTER;
+ this.unsetVariable(CLIENT_IP_VAR);
+ this.setVariable(URI_VAR, filter);
}
- this.emitChangeFilter();
- },
- getFilter: function () {
- return this.getRestFilter(filterName);
- },
- emitChangeFilter: function () {
- this.emit(CHANGE_FILTER_EVENT);
- },
- addChangeFilterListener: function (callback) {
- this.on(CHANGE_FILTER_EVENT, callback);
- },
- removeChangeFilterListener: function (callback) {
+ this.notifyListeners(CHANGE_FILTER_EVENT);
+ }
+
+ getFilter() {
+ return this.getVariable(CLIENT_IP_VAR) || this.getVariable(URI_VAR) || '';
+ }
+
+ addChangeFilterListener(callback) {
+ this.addListener(CHANGE_FILTER_EVENT, callback);
+ }
+
+ removeChangeFilterListener(callback) {
this.removeListener(CHANGE_FILTER_EVENT, callback);
- },
- highlightThreat: function (threat) {
- highlightedThread = threat;
- this.emitHighlightThreat();
- },
- getHighlightedThreat: function () {
- return highlightedThread;
- },
- addThreatHighlightListener: function (callback) {
- this.on(HIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatHighlightListener: function (callback) {
+ }
+
+ highlightThreat(threat) {
+ this.highlightedThread = threat;
+ this.notifyListeners(HIGHLIGHT_THREAT_EVENT);
+ }
+
+ getHighlightedThreat() {
+ return this.highlightedThread;
+ }
+
+ addThreatHighlightListener(callback) {
+ this.addListener(HIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatHighlightListener(callback) {
this.removeListener(HIGHLIGHT_THREAT_EVENT, callback);
- },
- emitHighlightThreat: function () {
- this.emit(HIGHLIGHT_THREAT_EVENT);
- },
- unhighlightThreat: function () {
- highlightedThread = null;
- this.emitUnhighlightThreat();
- },
- addThreatUnhighlightListener: function (callback) {
- this.on(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- removeThreatUnhighlightListener: function (callback) {
+ }
+
+ unhighlightThreat() {
+ this.highlightedThread = null;
+ this.notifyListeners(UNHIGHLIGHT_THREAT_EVENT);
+ }
+
+ addThreatUnhighlightListener(callback) {
+ this.addListener(UNHIGHLIGHT_THREAT_EVENT, callback);
+ }
+
+ removeThreatUnhighlightListener(callback) {
this.removeListener(UNHIGHLIGHT_THREAT_EVENT, callback);
- },
- emitUnhighlightThreat: function () {
- this.emit(UNHIGHLIGHT_THREAT_EVENT);
- },
- selectThreat: function (threat) {
- selectedThread = threat;
- this.emitThreatSelect();
- },
- getSelectedThreat: function () {
- return selectedThread;
- },
- addThreatSelectListener: function (callback) {
- this.on(SELECT_THREAT_EVENT, callback);
- },
- removeThreatSelectListener: function (callback) {
+ }
+
+ selectThreat(threat) {
+ this.selectedThread = threat;
+ this.notifyListeners(SELECT_THREAT_EVENT);
+ }
+
+ getSelectedThreat() {
+ return this.selectedThread;
+ }
+
+ addThreatSelectListener(callback) {
+ this.addListener(SELECT_THREAT_EVENT, callback);
+ }
+
+ removeThreatSelectListener(callback) {
this.removeListener(SELECT_THREAT_EVENT, callback);
- },
- emitThreatSelect: function () {
- this.emit(SELECT_THREAT_EVENT);
}
-});
+}
+
+const ss = new SuspiciousStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_FILTER:
- SuspiciousStore.setFilter(action.filter);
+ ss.setFilter(action.filter);
break;
case SpotConstants.UPDATE_DATE:
- SuspiciousStore.setDate(action.date);
+ ss.setDate(action.date);
break;
case SpotConstants.RELOAD_SUSPICIOUS:
- SuspiciousStore.reload();
+ ss.sendQuery();
break;
case SpotConstants.HIGHLIGHT_THREAT:
- SuspiciousStore.highlightThreat(action.threat);
+ ss.highlightThreat(action.threat);
break;
case SpotConstants.UNHIGHLIGHT_THREAT:
- SuspiciousStore.unhighlightThreat();
+ ss.unhighlightThreat();
break;
case SpotConstants.SELECT_THREAT:
- SuspiciousStore.selectThreat(action.threat);
+ ss.selectThreat(action.threat);
break;
}
});
-module.exports = SuspiciousStore;
+module.exports = ss;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/stores/TimelineStore.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/stores/TimelineStore.js b/spot-oa/ui/proxy/js/stores/TimelineStore.js
index fa6c704..cf8d886 100755
--- a/spot-oa/ui/proxy/js/stores/TimelineStore.js
+++ b/spot-oa/ui/proxy/js/stores/TimelineStore.js
@@ -1,61 +1,83 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
-
-const assign = require('object-assign');
-const d3 = require('d3');
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const SpotDispatcher = require('../../../js/dispatchers/SpotDispatcher');
-const FlowConstants = require('../constants/ProxyConstants');
const SpotConstants = require('../../../js/constants/SpotConstants');
-const RestStore = require('../../../js/stores/RestStore');
-
-const FILTER_NAME = 'hash';
-
-const TimelineStore = assign(new RestStore(FlowConstants.API_TIMELINE), {
- _parser: d3.tsv,
- _date:'',
- errorMessages: {
- 404: 'Please choose a different date, no data has been found'
- },
- setDate: function (date)
- {
- this.setEndpoint(FlowConstants.API_TIMELINE.replace('${date}', date.replace(/-/g, '')));
- this._date = date;
- },
+
+const ObservableGraphQLStore = require('../../../js/stores/ObservableGraphQLStore');
+
+const DATE_VAR = 'date';
+const URI_VAR = 'uri';
+
+class TimelineStore extends ObservableGraphQLStore {
+ getQuery() {
+ return `
+ query($date:SpotDateType!,$uri:String!) {
+ proxy {
+ threat {
+ timeline(date:$date, uri:$uri) {
+ duration
+ clientip: clientIp
+ tend: endDatetime
+ respcode: responseCodeLabel
+ tstart: startDatetime
+ }
+ }
+ }
+ }
+ `;
+ }
+
+ unboxData(data) {
+ return data.proxy.threat.timeline;
+ }
+
+ setDate(date) {
+ this.setVariable(DATE_VAR, date);
+ }
+
getDate() {
- return this._date;
- },
- setFilter: function (value)
- {
- this.setRestFilter(FILTER_NAME, value);
- },
- getFilter: function ()
- {
- return this.getRestFilter(FILTER_NAME);
- },
- clearFilter: function ()
- {
- this.removeRestFilter(FILTER_NAME);
+ return this.getVariable(DATE_VAR);
}
-});
+ setUri(uri) {
+ this.setVariable(URI_VAR, uri);
+ }
+
+ getUri() {
+ return this.getVariable(URI_VAR);
+ }
+}
+
+const ts = new TimelineStore();
SpotDispatcher.register(function (action) {
switch (action.actionType) {
case SpotConstants.UPDATE_DATE:
- TimelineStore.setDate(action.date);
-
+ ts.setDate(action.date);
break;
case SpotConstants.RELOAD_COMMENTS:
- TimelineStore.clearFilter();
- TimelineStore.resetData();
+ ts.resetData();
break;
case SpotConstants.SELECT_COMMENT:
- TimelineStore.setFilter(action.comment.hash);
-
- TimelineStore.reload();
-
+ ts.setUri(action.comment.uri);
+ ts.sendQuery();
break;
}
});
-module.exports = TimelineStore;
+module.exports = ts;
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/storyboard.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/storyboard.js b/spot-oa/ui/proxy/js/storyboard.js
index f642f30..8996fe1 100755
--- a/spot-oa/ui/proxy/js/storyboard.js
+++ b/spot-oa/ui/proxy/js/storyboard.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
require("babel-polyfill");
const React = require('react');
@@ -9,6 +24,12 @@ const SpotActions = require('../../js/actions/SpotActions');
const SpotConstants = require('../../js/constants/SpotConstants');
const SpotUtils = require('../../js/utils/SpotUtils');
const StoryboardActions = require('../../js/actions/StoryboardActions');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
ReactDOM.render(
(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/suspicious.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/suspicious.js b/spot-oa/ui/proxy/js/suspicious.js
index 95912c8..d38d11f 100755
--- a/spot-oa/ui/proxy/js/suspicious.js
+++ b/spot-oa/ui/proxy/js/suspicious.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -11,12 +26,22 @@ const SpotUtils = require('../../js/utils/SpotUtils');
// Build and Render Toolbar
const FilterInput = require('./components/FilterInput.react');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
function resetFilterAndReload() {
EdInActions.setFilter('');
EdInActions.reloadSuspicious();
};
+function switchComponents () {
+ SpotUtils.switchDivs(SpotConstants.DETAILS_PANEL, SpotConstants.SCORING_PANEL);
+};
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
+
ReactDOM.render(
(
<form className="form-inline">
@@ -56,11 +81,9 @@ const Panel = require('../../js/components/Panel.react');
const SuspiciousPanel = require('./components/SuspiciousPanel.react');
const NetworkViewPanel = require('./components/NetworkViewPanel.react');
-const IPythonNotebookPanel = require('../../js/components/IPythonNotebookPanel.react');
+const ScoreNotebook = require('./components/ScoreNotebook.react');
const DetailsPanel = require('./components/DetailsPanel.react');
-const ipynbClosure = IPythonNotebookPanel.createIPythonNotebookClosure(SpotConstants.NOTEBOOK_PANEL);
-
ReactDOM.render(
<div id="spot-content">
<PanelRow>
@@ -72,14 +95,18 @@ ReactDOM.render(
<NetworkViewPanel className="proxy-force" />
</Panel>
</PanelRow>
- <PanelRow>
- <Panel title={ipynbClosure.getTitle()} container extraButtons={ipynbClosure.getButtons}>
- <IPythonNotebookPanel title={ipynbClosure.getTitle()} date={SpotUtils.getCurrentDate()} ipynb="proxy/${date}/Edge_Investigation.ipynb" />
+ <div className="sortable">
+ <PanelRow title={SpotConstants.SCORING_PANEL}>
+ <Panel title={SpotConstants.SCORING_PANEL} reloadable switchable onReload={EdInActions.reloadSuspicious} onSwitch={switchComponents} className="col-md-12">
+ <ScoreNotebook />
</Panel>
- <Panel title={SpotConstants.DETAILS_PANEL} expandable>
- <DetailsPanel />
+ </PanelRow>
+ <PanelRow title={SpotConstants.DETAILS_PANEL}>
+ <Panel title={SpotConstants.DETAILS_PANEL} container switchable expandable onSwitch={switchComponents} className="col-md-12">
+ <DetailsPanel title={SpotConstants.DETAILS_PANEL} />
</Panel>
</PanelRow>
+ </div>
</div>,
document.getElementById('spot-content-wrapper')
);
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/js/threat-investigation.js
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/js/threat-investigation.js b/spot-oa/ui/proxy/js/threat-investigation.js
index 208f2ee..0701a32 100755
--- a/spot-oa/ui/proxy/js/threat-investigation.js
+++ b/spot-oa/ui/proxy/js/threat-investigation.js
@@ -1,4 +1,19 @@
-// Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements; and to You under the Apache License, Version 2.0.
+//
+// Licensed to the Apache Software Foundation (ASF) under one or more
+// contributor license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright ownership.
+// The ASF licenses this file to You under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
const React = require('react');
const ReactDOM = require('react-dom');
@@ -7,6 +22,12 @@ const SpotActions = require('../../js/actions/SpotActions');
const SpotUtils = require('../../js/utils/SpotUtils');
const DateInput = require('../../js/components/DateInput.react');
+const MainMenu = require('../../js/menu/components/MainMenu.react');
+
+ReactDOM.render(
+ <MainMenu />,
+ document.getElementById('main-menu')
+);
ReactDOM.render(
(
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/package.json
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/package.json b/spot-oa/ui/proxy/package.json
index bc3f4de..b986881 100755
--- a/spot-oa/ui/proxy/package.json
+++ b/spot-oa/ui/proxy/package.json
@@ -6,10 +6,12 @@
"watch-suspicious": "watchify js/suspicious.js -o js/suspicious.bundle.min.js -v -d",
"watch-threat-investigation": "watchify js/threat-investigation.js -o js/threat-investigation.bundle.min.js -v -d",
"watch-storyboard": "watchify js/storyboard.js -o js/storyboard.bundle.min.js -v -d",
- "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard",
+ "watch-notebooks": "watchify js/notebooks.js -o js/notebooks.bundle.min.js -v -d",
+ "build-all": "npm run build-suspicious && npm run build-threat-investigation && npm run build-storyboard && npm run build-notebooks",
"build-suspicious": "browserify js/suspicious.js | uglifyjs -cm > js/suspicious.bundle.min.js",
"build-threat-investigation": "browserify js/threat-investigation.js | uglifyjs -cm > js/threat-investigation.bundle.min.js",
- "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js"
+ "build-storyboard": "browserify js/storyboard.js | uglifyjs -cm > js/storyboard.bundle.min.js",
+ "build-notebooks": "browserify js/notebooks.js | uglifyjs -cm > js/notebooks.bundle.min.js"
},
"browserify": {
"transform": [
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/storyboard.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/storyboard.html b/spot-oa/ui/proxy/storyboard.html
index cabd387..01bdd94 100755
--- a/spot-oa/ui/proxy/storyboard.html
+++ b/spot-oa/ui/proxy/storyboard.html
@@ -21,6 +21,7 @@
<title>Proxy :: Storyboard</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -148,62 +149,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious Connects</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/suspicious.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/suspicious.html b/spot-oa/ui/proxy/suspicious.html
index 1d77322..29bea60 100755
--- a/spot-oa/ui/proxy/suspicious.html
+++ b/spot-oa/ui/proxy/suspicious.html
@@ -21,6 +21,7 @@
<title>Proxy :: Suspicious</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -30,6 +31,8 @@
<link rel="stylesheet" type="text/css" href="../node_modules/bootstrap-datepicker/dist/css/bootstrap-datepicker3.min.css" />
<!-- Spot styles -->
<link rel="stylesheet" type="text/css" href="../css/main.css" />
+ <!-- Sweetalert2 -->
+ <link rel="stylesheet" type="text/css" href="../node_modules/sweetalert2/dist/sweetalert2.min.css">
<!-- Favicon -->
<link rel="apple-touch-icon" sizes="57x57" href="../images/favicon/apple-icon-57x57.png">
<link rel="apple-touch-icon" sizes="60x60" href="../images/favicon/apple-icon-60x60.png">
@@ -138,62 +141,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
@@ -217,6 +165,7 @@
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom.min.js"></script>
<script type="application/javascript" src="../node_modules/react-dom/dist/react-dom-server.min.js"></script>
<script type="application/javascript" src="../js/tooltip.js"></script>
+ <script type="application/javascript" src="../node_modules/sweetalert2/dist/sweetalert2.min.js"></script>
<script type="application/javascript" src="js/suspicious.bundle.min.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/ui/proxy/threat-investigation.html
----------------------------------------------------------------------
diff --git a/spot-oa/ui/proxy/threat-investigation.html b/spot-oa/ui/proxy/threat-investigation.html
index 98c3ca4..9493f98 100755
--- a/spot-oa/ui/proxy/threat-investigation.html
+++ b/spot-oa/ui/proxy/threat-investigation.html
@@ -21,6 +21,7 @@
<title>Proxy :: Threat Investigation</title>
<meta charset="UTF-8">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=1.0, user-scalable=no" />
<!--Bootstrap styles-->
<link rel="stylesheet" type="text/css" href="../css/bootstrap-spot.min.css" />
@@ -67,62 +68,7 @@
</div>
<!-- Collect the nav links, forms, and other content for toggling -->
<div class="collapse navbar-collapse" id="main-menu">
- <ul class="nav navbar-nav navbar-right">
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Flows
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="flowsMenu">
- <li>
- <a data-href="../flow/suspicious.html#date=${date}">Suspicious Connects</a>
- </li>
- <li>
- <a data-href="../flow/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../flow/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- DNS
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="dnsMenu">
- <li>
- <a data-href="../dns/suspicious.html#date=${date}">Suspicious DNS</a>
- </li>
- <li>
- <a data-href="../dns/threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="../dns/storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li class="dropdown">
- <a class="dropdown-toggle" data-toggle="dropdown" role="button" aria-haspopup="true" aria-expanded="false">
- Proxy
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="proxyMenu">
- <li>
- <a data-href="suspicious.html#date=${date}">Suspicious</a>
- </li>
- <li>
- <a data-href="threat-investigation.html#date=${date}">Threat Investigation</a>
- </li>
- <li>
- <a data-href="storyboard.html#date=${date}">Storyboard</a>
- </li>
- </ul>
- </li>
- <li>
- <a data-href="../ingest-summary.html#end-date=${date}">Ingest Summary</a>
- </li>
- </ul>
+ <!-- Main Menu -->
</div>
<div id="search-box" class="row text-right">
<!--Tools Buttons-->
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/README.md b/spot-setup/README.md
index 4a80c1b..318f95e 100644
--- a/spot-setup/README.md
+++ b/spot-setup/README.md
@@ -8,48 +8,73 @@ This document is intended for any developer or sysadmin in learning the technica
This information will help you to get started on contributing to the Apache Spot Setup repository. For information about installing and running Apache Spot go to our [Installation Guide](http://spot.apache.org/doc/).
-Spot-setup contains the scripts to setup HDFS for Apache Spot solution. It will create the folder and database structure needed to run Apache Spot on HDFS and HIVE respectively. Spot-setup is a component of Apache Spot and is executed in the initial configuration after Linux user creation and before Ingest installation.
+Spot-setup contains the scripts to setup HDFS for Apache Spot solution. It will create the folder and database structure needed to run Apache Spot on HDFS and Impala respectively. Spot-setup is a component of Apache Spot and is executed in the initial configuration after Linux user creation and before Ingest installation.
## Prerequisites
To collaborate and run spot-setup, it is required the following prerequisites:
- A running Hadoop cluster
- Linux user account created in all nodes with sudo privileges
+- Enabling HDFS Access Control Lists
## General Description
-The main script in the repository is **hdfs_setup.sh** which is responsible of loading environment variables, creating folders in Hadoop for the different use cases (flow, DNS or Proxy), create the Hive database, and finally execute hive query scripts that creates Hive tables needed to access netflow, dns and proxy data.
+The main script in the repository is **hdfs_setup.sh** which is responsible of loading environment variables, creating folders in Hadoop for the different use cases (flow, DNS or Proxy), create the Impala database, and finally execute Impala query scripts that creates Impala tables needed to access netflow, dns and proxy data.
## Environment Variables
**spot.conf** is the file storing the variables needed during the installation process including node assignment, User interface, Machine Learning and Ingest gateway nodes.
-This file also contains sources desired to be installed as part of Apache Spot, general paths for HDFS folders, Kerberos information and local paths in the Linux filesystem for the user as well as for machine learning, ipython, lda and ingest processes.
+This file also contains sources desired to be installed as part of Apache Spot, general paths for HDFS folders, and local paths in the Linux filesystem for the user as well as for machine learning, ipython, lda and ingest processes.
-To read more about these variables, please review the [wiki] (http://spot.incubator.apache.org/doc/#configuration).
+To read more about these variables, please review the [documentation](http://spot.incubator.apache.org/doc/#configuration).
## Database Query Scripts
spot-setup contains a script per use case, as of today, there is a table creation script for each DNS, flow and Proxy data.
-These HQL scripts are intended to be executed as a Hive statement and must comply HQL standards.
+These HQL scripts are intended to be executed as a Impala statement and must comply HQL standards.
-We want to create tables in Avro/Parquet format to get a faster query performance. This format is an industry standard and you can find more information about it on:
-- Avro is a data serialization system - https://avro.apache.org/
+We create tables using Parquet format to get a faster query performance. This format is an industry standard and you can find more information about it on:
- Parquet is a columnar storage format - https://parquet.apache.org/
-To get to Avro/parquet format we need a staging table to store CSV data temporarily for Flow and DNS. Then, run a Hive query statement to insert these text-formatted records into the Avro/parquet table. Hive will manage to convert the text data into the desired format. The staging table must be cleaned after loading data to Avro/parquet table for the next batch cycle. For Flow and DNS, a set of a staging (CSV) and a final (Avro/parquet) tables are needed for each data entity. For Proxy, only the Avro/parquet table is needed.
+To get to parquet format we need a staging table to store CSV data temporarily for Flow and DNS. Then, run an Impala query statement to insert these text-formatted records into the parquet table. Impala will manage to convert the text data into the desired format. The staging table must be cleaned after loading data to parquet table for the next batch cycle. For Flow and DNS, a set of a staging (CSV) and a final (parquet) tables are needed for each data entity. For Proxy, only the parquet table is needed.
#### Flow Tables
-- flow - Avro/parquet final table to store flow records
-- flow_tmp - Text table to store temporarily flow records in CSV format
+- flow
+- flow_tmp
+- flow_chords
+- flow_edge
+- flow_ingest_summary
+- flow_scores
+- flow_storyboard
+- flow_threat_investigation
+- flow_timeline
#### DNS Tables
-- dns - Avro/parquet final table to store DNS records
-- dns_tmp - Text table to store temporarily DNS records in CSV format
+- dns
+- dns_tmp
+- dns_dendro
+- dns_edge
+- dns_ingest_summary
+- dns_scores
+- dns_storyboard
+- dns_threat_dendro
+- dns_threat_investigation
#### Proxy Tables
-- proxy - Avro/parquet final table to store Proxy records
+- proxy
+- proxy_edge
+- proxy_ingest_summary
+- proxy_scores
+- proxy_storyboard
+- proxy_threat_investigation
+- proxy_timeline
+
+
+## Migrating OA Data to Spot 1.0
+
+Please review migration documentation [here](migration/README.md).
## Licensing
@@ -61,10 +86,15 @@ Create a pull request and contact the maintainers.
## Issues
+<<<<<<< HEAD
Report issues at the Apache Spot [issues] (https://issues.apache.org/jira/projects/SPOT/issues) page.
+=======
+- Create an [issue](https://issues.apache.org/jira/browse/SPOT-20?jql=project%20%3D%20SPOT).
+- Go to our Slack [channel](https://apachespot.slack.com/messages/general).
+>>>>>>> 2ebe57238614b6aa1162c6dc30758c516ae79273
## Maintainers
-- [Moises Valdovinos] (https://github.com/moy8011)
-- [Everardo Lopez Sandoval] (https://github.com/EverLoSa)
+- [Moises Valdovinos](https://github.com/moy8011)
+- [Everardo Lopez Sandoval](https://github.com/EverLoSa)
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_dns_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_dns_avro_parquet.hql b/spot-setup/create_dns_avro_parquet.hql
deleted file mode 100755
index 105c386..0000000
--- a/spot-setup/create_dns_avro_parquet.hql
+++ /dev/null
@@ -1,37 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.dns (
- frame_time STRING,
- unix_tstamp BIGINT,
- frame_len INT,
- ip_dst STRING,
- ip_src STRING,
- dns_qry_name STRING,
- dns_qry_class STRING,
- dns_qry_type INT,
- dns_qry_rcode INT,
- dns_a STRING
-)
-PARTITIONED BY (y INT, m INT, d INT, h int)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/dns/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "DnsRecord"
- , "namespace" : "com.cloudera.accelerators.dns.avro"
- , "fields": [
- {"name": "frame_time", "type":["string", "null"]}
- , {"name": "unix_tstamp", "type":["bigint", "null"]}
- , {"name": "frame_len", "type":["int", "null"]}
- , {"name": "ip_dst", "type":["string", "null"]}
- , {"name": "ip_src", "type":["string", "null"]}
- , {"name": "dns_qry_name", "type":["string", "null"]}
- , {"name": "dns_qry_class", "type":["string", "null"]}
- , {"name": "dns_qry_type", "type":["int", "null"]}
- , {"name": "dns_qry_rcode", "type":["int", "null"]}
- , {"name": "dns_a", "type":["string", "null"]}
- ]
-}');
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_dns_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_dns_parquet.hql b/spot-setup/create_dns_parquet.hql
new file mode 100755
index 0000000..38025c6
--- /dev/null
+++ b/spot-setup/create_dns_parquet.hql
@@ -0,0 +1,163 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.dns (
+frame_time STRING,
+unix_tstamp BIGINT,
+frame_len INT,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+dns_a STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT,
+h TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro (
+unix_tstamp BIGINT,
+dns_a STRING,
+dns_qry_name STRING,
+ip_dst STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/dendro';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_edge (
+unix_tstamp BIGINT,
+frame_len BIGINT,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+dns_a STRING,
+hh INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_scores (
+frame_time STRING,
+unix_tstamp BIGINT,
+frame_len BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+ml_score FLOAT,
+tld STRING,
+query_rep STRING,
+hh INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard (
+ip_threat STRING,
+dns_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro (
+anchor STRING,
+total BIGINT,
+dns_qry_name STRING,
+ip_dst STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/threat_dendro';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_investigation (
+unix_tstamp BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+ip_sev INT,
+dns_sev INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/dns/hive/oa/threat_investigation';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_flow_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_flow_avro_parquet.hql b/spot-setup/create_flow_avro_parquet.hql
deleted file mode 100755
index 34aa63b..0000000
--- a/spot-setup/create_flow_avro_parquet.hql
+++ /dev/null
@@ -1,72 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.flow (
- treceived STRING,
- unix_tstamp BIGINT,
- tryear INT,
- trmonth INT,
- trday INT,
- trhour INT,
- trminute INT,
- trsec INT,
- tdur FLOAT,
- sip STRING,
- dip STRING,
- sport INT,
- dport INT,
- proto STRING,
- flag STRING,
- fwd INT,
- stos INT,
- ipkt BIGINT,
- ibyt BIGINT,
- opkt BIGINT,
- obyt BIGINT,
- input INT,
- output INT,
- sas INT,
- das INT,
- dtos INT,
- dir INT,
- rip STRING
- )
-PARTITIONED BY (y INT, m INT, d INT, h int)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/flow/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "FlowRecord"
- , "namespace" : "com.cloudera.accelerators.flows.avro"
- , "fields": [
- {"name": "treceived", "type":["string", "null"]}
- , {"name": "unix_tstamp", "type":["long", "null"]}
- , {"name": "tryear", "type":["int", "null"]}
- , {"name": "trmonth", "type":["int", "null"]}
- , {"name": "trday", "type":["int", "null"]}
- , {"name": "trhour", "type":["int", "null"]}
- , {"name": "trminute", "type":["int", "null"]}
- , {"name": "trsec", "type":["int", "null"]}
- , {"name": "tdur", "type":["float", "null"]}
- , {"name": "sip", "type":["string", "null"]}
- , {"name": "sport", "type":["int", "null"]}
- , {"name": "dip", "type":["string", "null"]}
- , {"name": "dport", "type":["int", "null"]}
- , {"name": "proto", "type":["string", "null"]}
- , {"name": "flag", "type":["string", "null"]}
- , {"name": "fwd", "type":["int", "null"]}
- , {"name": "stos", "type":["int", "null"]}
- , {"name": "ipkt", "type":["bigint", "null"]}
- , {"name": "ibytt", "type":["bigint", "null"]}
- , {"name": "opkt", "type":["bigint", "null"]}
- , {"name": "obyt", "type":["bigint", "null"]}
- , {"name": "input", "type":["int", "null"]}
- , {"name": "output", "type":["int", "null"]}
- , {"name": "sas", "type":["int", "null"]}
- , {"name": "das", "type":["int", "null"]}
- , {"name": "dtos", "type":["int", "null"]}
- , {"name": "dir", "type":["int", "null"]}
- , {"name": "rip", "type":["string", "null"]}
- ]
-}');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_flow_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_flow_parquet.hql b/spot-setup/create_flow_parquet.hql
new file mode 100755
index 0000000..41c4819
--- /dev/null
+++ b/spot-setup/create_flow_parquet.hql
@@ -0,0 +1,195 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.flow (
+treceived STRING,
+unix_tstamp BIGINT,
+tryear INT,
+trmonth INT,
+trday INT,
+trhour INT,
+trminute INT,
+trsec INT,
+tdur FLOAT,
+sip STRING,
+dip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flag STRING,
+fwd INT,
+stos INT,
+ipkt BIGINT,
+ibyt BIGINT,
+opkt BIGINT,
+obyt BIGINT,
+input INT,
+output INT,
+sas INT,
+das INT,
+dtos INT,
+dir INT,
+rip STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT,
+h TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_chords (
+ip_threat STRING,
+srcip STRING,
+dstip STRING,
+ibyt BIGINT,
+ipkt BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/chords';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_edge (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flags STRING,
+tos INT,
+ibyt BIGINT,
+ipkt BIGINT,
+input BIGINT,
+output BIGINT,
+rip STRING,
+obyt BIGINT,
+opkt BIGINT,
+hh INT,
+mn INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_scores (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+ipkt INT,
+ibyt INT,
+opkt INT,
+obyt INT,
+ml_score FLOAT,
+rank INT,
+srcip_INTernal INT,
+dstip_INTernal INT,
+src_geoloc STRING,
+dst_geoloc STRING,
+src_domain STRING,
+dst_domain STRING,
+src_rep STRING,
+dst_rep STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard (
+ip_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_threat_investigation (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+srcport INT,
+dstport INT,
+score INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/threat_investigation';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline (
+ip_threat STRING,
+tstart STRING,
+tend STRING,
+srcip STRING,
+dstip STRING,
+proto STRING,
+sport INT,
+dport INT,
+ipkt BIGINT,
+ibyt BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/flow/hive/oa/timeline';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_proxy_avro_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_proxy_avro_parquet.hql b/spot-setup/create_proxy_avro_parquet.hql
deleted file mode 100755
index 3fd7ea8..0000000
--- a/spot-setup/create_proxy_avro_parquet.hql
+++ /dev/null
@@ -1,76 +0,0 @@
-SET hiveconf:huser;
-SET hiveconf:dbname;
-
-CREATE EXTERNAL TABLE IF NOT EXISTS ${hiveconf:dbname}.proxy (
-p_date string,
-p_time string,
-clientip string,
-host string,
-reqmethod string,
-useragent string,
-resconttype string,
-duration int,
-username string,
-authgroup string,
-exceptionid string,
-filterresult string,
-webcat string,
-referer string,
-respcode string,
-action string,
-urischeme string,
-uriport string,
-uripath string,
-uriquery string,
-uriextension string,
-serverip string,
-scbytes int,
-csbytes int,
-virusid string,
-bcappname string,
-bcappoper string,
-fulluri string
-)
-PARTITIONED BY (
-y string,
-m string,
-d string,
-h string)
-ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
-STORED AS PARQUET
-LOCATION '${hiveconf:huser}/proxy/hive'
-TBLPROPERTIES ('avro.schema.literal'='{
- "type": "record"
- , "name": "ProxyRecord"
- , "namespace" : "com.cloudera.accelerators.proxy.avro"
- , "fields": [
- {"name": "p_date", "type":["string", "null"]}
- , {"name": "p_time", "type":["string", "null"]}
- , {"name": "clientip", "type":["string", "null"]}
- , {"name": "host", "type":["string", "null"]}
- , {"name": "reqmethod", "type":["string", "null"]}
- , {"name": "useragent", "type":["string", "null"]}
- , {"name": "resconttype", "type":["string", "null"]}
- , {"name": "duration", "type":["int", "null"]}
- , {"name": "username", "type":["string", "null"]}
- , {"name": "authgroup", "type":["string", "null"]}
- , {"name": "exceptionid", "type":["string", "null"]}
- , {"name": "filterresult", "type":["string", "null"]}
- , {"name": "webcat", "type":["string", "null"]}
- , {"name": "referer", "type":["string", "null"]}
- , {"name": "respcode", "type":["string", "null"]}
- , {"name": "action", "type":["string", "null"]}
- , {"name": "urischeme", "type":["string", "null"]}
- , {"name": "uriport", "type":["string", "null"]}
- , {"name": "uripath", "type":["string", "null"]}
- , {"name": "uriquery", "type":["string", "null"]}
- , {"name": "uriextension", "type":["string", "null"]}
- , {"name": "serverip", "type":["string", "null"]}
- , {"name": "scbytes", "type":["int", "null"]}
- , {"name": "csbytes", "type":["int", "null"]}
- , {"name": "virusid", "type":["string", "null"]}
- , {"name": "bcappname", "type":["string", "null"]}
- , {"name": "bcappoper", "type":["string", "null"]}
- , {"name": "fulluri", "type":["string", "null"]}
- ]
-}');
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/create_proxy_parquet.hql
----------------------------------------------------------------------
diff --git a/spot-setup/create_proxy_parquet.hql b/spot-setup/create_proxy_parquet.hql
new file mode 100755
index 0000000..f665dc2
--- /dev/null
+++ b/spot-setup/create_proxy_parquet.hql
@@ -0,0 +1,177 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE EXTERNAL TABLE IF NOT EXISTS ${var:dbname}.proxy (
+p_date STRING,
+p_time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+authgroup STRING,
+exceptionid STRING,
+filterresult STRING,
+webcat STRING,
+referer STRING,
+respcode STRING,
+action STRING,
+urischeme STRING,
+uriport STRING,
+uripath STRING,
+uriquery STRING,
+uriextension STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+virusid STRING,
+bcappname STRING,
+bcappoper STRING,
+fulluri STRING
+)
+PARTITIONED BY (
+y STRING,
+m STRING,
+d STRING,
+h STRING
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+webcat STRING,
+respcode STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+referer STRING,
+uriport STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+hh INT,
+respcode_name STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/edge';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary (
+tdate STRING,
+total BIGINT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/summary';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+webcat STRING,
+referer STRING,
+respcode INT,
+uriport INT,
+uripath STRING,
+uriquery STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+word STRING,
+ml_score FLOAT,
+uri_rep STRING,
+respcode_name STRING,
+network_context STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/suspicious';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard (
+p_threat STRING,
+title STRING,
+text STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/storyboard';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_threat_investigation (
+tdate STRING,
+fulluri STRING,
+uri_sev INT
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/threat_investigation';
+
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline (
+p_threat STRING,
+tstart STRING,
+tend STRING,
+duration BIGINT,
+clientip STRING,
+respcode STRING,
+respcodename STRING
+)
+PARTITIONED BY (
+y SMALLINT,
+m TINYINT,
+d TINYINT
+)
+STORED AS PARQUET
+LOCATION '${var:huser}/proxy/hive/oa/timeline';
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/hdfs_setup.sh
----------------------------------------------------------------------
diff --git a/spot-setup/hdfs_setup.sh b/spot-setup/hdfs_setup.sh
index 8eb9ae3..86a26c0 100755
--- a/spot-setup/hdfs_setup.sh
+++ b/spot-setup/hdfs_setup.sh
@@ -18,35 +18,52 @@
#
DSOURCES=('flow' 'dns' 'proxy')
-DFOLDERS=('binary' 'hive' 'stage')
+DFOLDERS=('binary'
+'stage'
+'hive'
+'hive/oa'
+'hive/oa/chords'
+'hive/oa/edge'
+'hive/oa/summary'
+'hive/oa/suspicious'
+'hive/oa/storyboard'
+'hive/oa/threat_investigation'
+'hive/oa/timeline'
+'hive/oa/dendro'
+'hive/oa/threat_dendro'
+)
+
+# Sourcing spot configuration variables
source /etc/spot.conf
-#
-# creating HDFS user's folder
-#
-hadoop fs -mkdir ${HUSER}
-hadoop fs -chown ${USER}:supergroup ${HUSER}
+# Creating HDFS user's folder
+sudo -u hdfs hdfs dfs -mkdir ${HUSER}
+sudo -u hdfs hdfs dfs -chown ${USER}:supergroup ${HUSER}
+sudo -u hdfs hdfs dfs -chmod 775 ${HUSER}
+# Creating HDFS paths for each use case
for d in "${DSOURCES[@]}"
do
echo "creating /$d"
- hadoop fs -mkdir ${HUSER}/$d
+ hdfs dfs -mkdir ${HUSER}/$d
for f in "${DFOLDERS[@]}"
do
echo "creating $d/$f"
- hadoop fs -mkdir ${HUSER}/$d/$f
+ hdfs dfs -mkdir ${HUSER}/$d/$f
done
+
+ # Modifying permission on HDFS folders to allow Impala to read/write
+ hdfs dfs -chmod -R 775 ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:impala:rwx ${HUSER}/$d
+ sudo -u hdfs hdfs dfs -setfacl -R -m user:${USER}:rwx ${HUSER}/$d
done
-#
-# create hive tables
-#
-#configure / create catalog
-hive -e "CREATE DATABASE ${DBNAME}"
+# Creating Spot Database
+impala-shell -i ${IMPALA_DEM} -q "CREATE DATABASE IF NOT EXISTS ${DBNAME};"
+# Creating Impala tables
for d in "${DSOURCES[@]}"
do
- hive -hiveconf huser=${HUSER} -hiveconf dbname=${DBNAME} -f create_${d}_avro_parquet.hql
+ impala-shell -i ${IMPALA_DEM} --var=huser=${HUSER} --var=dbname=${DBNAME} -c -f create_${d}_parquet.hql
done
-
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migrate_to_spot_1_0.py
----------------------------------------------------------------------
diff --git a/spot-setup/migrate_to_spot_1_0.py b/spot-setup/migrate_to_spot_1_0.py
new file mode 100755
index 0000000..1c00a4f
--- /dev/null
+++ b/spot-setup/migrate_to_spot_1_0.py
@@ -0,0 +1,102 @@
+#!/bin/env python
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+import subprocess
+from migration.utilities import util
+
+pipelines = sys.argv[1]
+old_oa_path = sys.argv[2]
+staging_db = sys.argv[3]
+hdfs_staging_path = sys.argv[4]
+dest_db = sys.argv[5]
+impala_daemon = sys.argv[6]
+
+
+# Execution example:
+#./migrate_old_data.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+
+def main():
+ log = util.get_logger('SPOT.MIGRATE')
+
+ cur_path = os.path.dirname(os.path.realpath(__file__))
+
+ global old_oa_path
+ old_oa_path = old_oa_path.rstrip('/')
+ old_spot_path = os.path.split(old_oa_path)[0]
+ new_spot_path = os.path.split(cur_path)[0]
+ new_oa_path = '{0}/spot-oa'.format(new_spot_path)
+
+ log.info('Old Spot path: {0}'.format(old_spot_path))
+ log.info('Old OA path: {0}'.format(old_oa_path))
+ log.info('New Spot path: {0}'.format(new_spot_path))
+ log.info('New OA path: {0}'.format(new_oa_path))
+
+
+ log.info("Executing hdfs_setup.sh for initial setup")
+ util.execute_cmd('./hdfs_setup.sh',log)
+
+ log.info("Updating /etc/spot.conf with new changes in Spot 1.0")
+ util.execute_cmd("migration/spot_conf_migration.py '/etc/spot.conf' '{0}/spot.conf'".format(cur_path),log)
+
+ log.info("Copy ingest_conf.json file from old folder code to new Spot location")
+ util.execute_cmd('cp {0}/spot-ingest/ingest_conf.json {1}/spot-ingest/ingest_conf.json'.format(old_spot_path, new_spot_path),log)
+
+ log.info("Copy Spot OA configuration files to new Spot location")
+ util.execute_cmd('cp {0}/oa/components/iana/iana_config.json {1}/oa/components/iana/iana_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/reputation/reputation_config.json {1}/oa/components/reputation/reputation_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/nc/nc_config.json {1}/oa/components/nc/nc_config.json'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/oa/components/data/engine.json {1}/oa/components/data/engine.json'.format(old_oa_path, new_oa_path),log)
+
+ log.info("Copy Spot OA context files to new Spot location")
+ util.execute_cmd('cp {0}/context/iploc.csv {1}/context/iploc.csv'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/context/ipranges.csv {1}/context/ipranges.csv'.format(old_oa_path, new_oa_path),log)
+ util.execute_cmd('cp {0}/context/networkcontext_1.csv {1}/context/networkcontext_1.csv'.format(old_oa_path, new_oa_path),log)
+
+ log.info("Install browserify and uglifyjs using npm")
+ os.chdir('{0}/ui'.format(new_oa_path))
+ util.execute_cmd('sudo -H -E npm install -g browserify uglifyjs',log)
+ util.execute_cmd('npm install',log)
+
+ log.info("Install python-devel using yum")
+ util.execute_cmd('sudo yum install python-devel -y',log)
+
+ log.info("Install requirements using pip")
+ os.chdir('{0}'.format(new_oa_path))
+ util.execute_cmd('sudo -H -E pip install -r requirements.txt',log)
+
+
+ log.info("Migrate data from pipelines")
+ l_pipelines = pipelines.split(',')
+ valid_pipelines = ('flow', 'dns', 'proxy')
+ if len(l_pipelines) > 0:
+ for pipe in l_pipelines:
+ if pipe in valid_pipelines:
+ log.info("Migrating {0} old data to new spot release".format(pipe))
+ os.chdir('{0}/spot-setup/migration'.format(new_spot_path))
+ util.execute_cmd("python migrate_old_{0}_data.py '{1}' '{2}' '{3}' '{4}' '{5}' ".format(pipe, old_oa_path, staging_db, hdfs_staging_path, dest_db, impala_daemon),log)
+ else:
+ log.error("Pipeline {0} is not valid. Valid pipelines are flow, dns or proxy".format(pipe))
+ else:
+ log.error("Pipeline arguments must be separated by commas")
+
+
+if __name__=='__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/README.md
----------------------------------------------------------------------
diff --git a/spot-setup/migration/README.md b/spot-setup/migration/README.md
new file mode 100644
index 0000000..29314ef
--- /dev/null
+++ b/spot-setup/migration/README.md
@@ -0,0 +1,71 @@
+## OA Data Migration to Spot 1.0
+
+This document is intended for any developer or sysadmin who wants to migrate their existing OA data to Spot 1.0. In previous Spot releases, OA data was stored in CSV files in a given location in the server used for OA (specified in spot.conf during original installation). In Spot 1.0, OA data is stored in Impala tables. The purposes of these scripts are to migrate independently each use case (flow, proxy and dns) from those CSV files into the new Impala tables.
+
+This migration process is optional and only for those users who want to keep usable their OA data (scores, edges, chords, dendros, ingest summaries, storyboards, threat investigations and timelines) generated in the previous Spot version.
+
+### Requirements
+
+- You must run first new Spot setup installation to have the new tables created in Impala.
+- You must log in to the OA server and run these scripts from there.
+- You must run these scripts from the spot-setup/migration folder in the new Spot 1.0 location
+
+### CSV to Impala Tables Mapping
+
+**Flow**
+
+CSV File | Impala Table
+---------|-------
+flow_scores.csv | flow_scores
+chord-*.tsv | flow_chords
+edge-*.tsv | flow_edge
+is_*.csv | flow_ingest_summary
+threats.csv | flow_storyboard
+flow_scores.csv (only scored values) | flow_threat_investigation
+sbdet-*.tsv | flow_timeline
+
+**DNS**
+
+CSV File | Impala Table
+---------|-------
+flow_scores.csv | dns_scores
+edge-*.csv | dns_edge
+dendro-*.csv | dns_dendro
+threat-dendro-*.csv | dns_threat_dendro
+is_*.csv | dns_ingest_summary
+threats.csv | dns_storyboard
+dns_scores.csv (only scored values) | dns_threat_investigation
+
+**Proxy**
+
+CSV File | Impala Table
+---------|-------
+edge-*.csv | proxy_edge
+is_*.csv | proxy_ingest_summary
+proxy_scores.csv | proxy_scores
+threats.csv | proxy_storyboard
+proxy_scores.csv (only scored values) | proxy_threat_investigation
+timeline-*.csv | proxy_timeline
+
+### Data Flow
+
+There is a launch and single script that will migrate all specified pipelines. This process will read each of the CSV from the existing location and import data to Impala tables accordingly, creating first a staging database and tables to load the records in the CSV and then insert that data into the new Spot 1.0 tables. You must execute this migration process from the server where old Spot release is located. You may provide one pipeline or all (flow, dns and proxy) according to your needs and your existing data. At the end of each script, the old data pipeline folder will be moved from the original location to a backup folder. Staging tables and their respective HDFS paths will be removed.
+
+### Execution
+
+```python
+./migrate_to_spot_1_0.py PIPELINES OLD_OA_PATH STAGING_DB_NAME STAGING_DB_HDFS_PATH NEW_SPOT_IMPALA_DB IMPALA_DAEMON
+```
+
+where variables mean:
+- **PIPELINES** - Comma-separated list of the pipelines to be migrated
+- **OLD_OA_PATH** - Path to the old Spot-OA release directory in the local filesystem
+- **STAGING_DB_NAME** - Name of the staging database to be created to temporarily store these records
+- **STAGING_DB_HDFS_PATH** - HDFS path of the staging database to be created to temporarily store these records
+- **NEW_SPOT_IMPALA_DB** - Database name of the Spot 1.0 Impala tables. Use the same as in the spot.conf when the new Spot release was installed
+- **IMPALA_DAEMON** - Choose an Impala daemon to be used to run scripts' queries.
+
+Example:
+```python
+./migrate_to_spot_1_0.py 'flow,dns,proxy' '/home/spotuser/incubator-spot_old/spot-oa' 'spot_migration' '/user/spotuser/spot_migration/' 'migrated' 'node01'
+```
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/__init__.py
----------------------------------------------------------------------
diff --git a/spot-setup/migration/__init__.py b/spot-setup/migration/__init__.py
new file mode 100644
index 0000000..ecb1860
--- /dev/null
+++ b/spot-setup/migration/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/create_dns_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_dns_migration_tables.hql b/spot-setup/migration/create_dns_migration_tables.hql
new file mode 100644
index 0000000..9afe6ba
--- /dev/null
+++ b/spot-setup/migration/create_dns_migration_tables.hql
@@ -0,0 +1,113 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE DATABASE IF NOT EXISTS ${var:dbname};
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_dendro_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_dendro_tmp (
+dns_a STRING,
+dns_qry_name STRING,
+ip_dst STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/dendro'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_edge_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_edge_tmp (
+frame_time STRING,
+frame_len STRING,
+ip_dst STRING,
+ip_src STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type STRING,
+dns_qry_rcode STRING,
+dns_a STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/edge'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_ingest_summary_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_ingest_summary_tmp (
+tdate STRING,
+total BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/summary'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_scores_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_scores_tmp (
+frame_time STRING,
+frame_len BIGINT,
+ip_dst STRING,
+dns_qry_name STRING,
+dns_qry_class STRING,
+dns_qry_type INT,
+dns_qry_rcode INT,
+ml_score FLOAT,
+tld STRING,
+query_rep STRING,
+hh INT,
+ip_sev INT,
+dns_sev INT,
+dns_qry_class_name STRING,
+dns_qry_type_name STRING,
+dns_qry_rcode_name STRING,
+network_context STRING,
+unix_tstamp BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/scores'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_storyboard_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_storyboard_tmp (
+ip_threat STRING,
+dns_threat STRING,
+title STRING,
+text STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+LOCATION '${var:hpath}/dns/storyboard'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.dns_threat_dendro_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.dns_threat_dendro_tmp (
+total BIGINT,
+dns_qry_name STRING,
+ip_dst STRING,
+sev int
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/dns/threat_dendro'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/create_flow_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_flow_migration_tables.hql b/spot-setup/migration/create_flow_migration_tables.hql
new file mode 100644
index 0000000..b200efb
--- /dev/null
+++ b/spot-setup/migration/create_flow_migration_tables.hql
@@ -0,0 +1,126 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE DATABASE IF NOT EXISTS ${var:dbname};
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_scores_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_scores_tmp (
+sev int,
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+ipkt INT,
+ibyt INT,
+opkt INT,
+obyt INT,
+score FLOAT,
+rank INT,
+srcIpInternal INT,
+destIpInternal INT,
+srcGeo STRING,
+dstGeo STRING,
+srcDomain STRING,
+dstDomain STRING,
+srcIP_rep STRING,
+dstIP_rep STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/flow/scores/'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_chords_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_chords_tmp (
+srcip STRING,
+dstip STRING,
+ibyt BIGINT,
+ipkt BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/flow/chords'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_edge_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_edge_tmp (
+tstart STRING,
+srcip STRING,
+dstip STRING,
+sport INT,
+dport INT,
+proto STRING,
+flags STRING,
+tos INT,
+ibyt BIGINT,
+ipkt BIGINT,
+input BIGINT,
+output BIGINT,
+rip STRING,
+obyt BIGINT,
+opkt BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/flow/edge'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_ingest_summary_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_ingest_summary_tmp (
+tdate STRING,
+total BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/flow/summary'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_storyboard_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_storyboard_tmp (
+ip_threat STRING,
+title STRING,
+text STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+LOCATION '${var:hpath}/flow/storyboard'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.flow_timeline_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.flow_timeline_tmp (
+tstart STRING,
+tend STRING,
+srcip STRING,
+dstip STRING,
+proto STRING,
+sport INT,
+dport INT,
+ipkt BIGINT,
+ibyt BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/flow/timeline'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-setup/migration/create_proxy_migration_tables.hql
----------------------------------------------------------------------
diff --git a/spot-setup/migration/create_proxy_migration_tables.hql b/spot-setup/migration/create_proxy_migration_tables.hql
new file mode 100644
index 0000000..fa92ab2
--- /dev/null
+++ b/spot-setup/migration/create_proxy_migration_tables.hql
@@ -0,0 +1,127 @@
+
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+
+-- http://www.apache.org/licenses/LICENSE-2.0
+
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+
+CREATE DATABASE IF NOT EXISTS ${var:dbname};
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_edge_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_edge_tmp (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+webcat STRING,
+respcode STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+referer STRING,
+uriport STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/proxy/edge'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_ingest_summary_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_ingest_summary_tmp (
+tdate STRING,
+total BIGINT
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/proxy/summary'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_scores_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_scores_tmp (
+tdate STRING,
+time STRING,
+clientip STRING,
+host STRING,
+reqmethod STRING,
+useragent STRING,
+resconttype STRING,
+duration INT,
+username STRING,
+webcat STRING,
+referer STRING,
+respcode INT,
+uriport INT,
+uripath STRING,
+uriquery STRING,
+serverip STRING,
+scbytes INT,
+csbytes INT,
+fulluri STRING,
+word STRING,
+ml_score FLOAT,
+uri_rep STRING,
+uri_sev INT,
+respcode_name STRING,
+network_context STRING,
+score_hash STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/proxy/scores'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_storyboard_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_storyboard_tmp (
+p_threat STRING,
+title STRING,
+text STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|'
+LOCATION '${var:hpath}/proxy/storyboard'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_timeline_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_timeline_tmp (
+tstart STRING,
+tend STRING,
+duration BIGINT,
+clientip STRING,
+respcode STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t'
+LOCATION '${var:hpath}/proxy/timeline'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+
+DROP TABLE IF EXISTS ${var:dbname}.proxy_iana_rcode_tmp;
+
+CREATE EXTERNAL TABLE ${var:dbname}.proxy_iana_rcode_tmp (
+respcode STRING,
+respcode_name STRING
+)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY ','
+LOCATION '${var:hpath}/proxy/iana_rcode'
+TBLPROPERTIES ('skip.header.line.count'='1');
+
+