You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spot.apache.org by na...@apache.org on 2017/09/26 21:37:18 UTC

[12/18] incubator-spot git commit: syncing with master

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/proxy/query.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/proxy/query.py b/spot-oa/api/graphql/proxy/query.py
new file mode 100644
index 0000000..1e32f34
--- /dev/null
+++ b/spot-oa/api/graphql/proxy/query.py
@@ -0,0 +1,582 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from datetime import date
+from graphql import (
+    GraphQLObjectType,
+    GraphQLArgument,
+    GraphQLNonNull,
+    GraphQLField,
+    GraphQLString,
+    GraphQLInt,
+    GraphQLList
+)
+
+from api.graphql.common import SpotDateType, SpotDatetimeType, SpotIpType, IngestSummaryType
+import api.resources.proxy as Proxy
+
+SuspiciousType = GraphQLObjectType(
+    name='ProxySuspiciousType',
+    fields={
+        'datetime': GraphQLField(
+            type=SpotDatetimeType,
+            description='Start time of the request',
+            resolver=lambda root, *_: '{} {}'.format(root.get('tdate') or '1970-01-01', root.get('time') or '00:00:00')
+        ),
+        'clientIp': GraphQLField(
+            type=SpotIpType,
+            description='Client\'s IP address',
+            resolver=lambda root, *_: root.get('clientip')
+        ),
+        'host': GraphQLField(
+            type=GraphQLString,
+            description='Host name from the client request URL',
+            resolver=lambda root, *_: root.get('host')
+        ),
+        'requestMethod': GraphQLField(
+            type=GraphQLString,
+            description='HTTP request method',
+            resolver=lambda root, *_: root.get('reqmethod')
+        ),
+        'userAgent': GraphQLField(
+            type=GraphQLString,
+            description='Client\'s user agent',
+            resolver=lambda root, *_: root.get('useragent')
+        ),
+        'responseContentType': GraphQLField(
+            type=GraphQLString,
+            description='HTTP response content type (MIME)',
+            resolver=lambda root, *_: root.get('resconttype')
+        ),
+        'duration': GraphQLField(
+            type=GraphQLInt,
+            description='Connection duration',
+            resolver=lambda root, *_: root.get('duration')
+        ),
+        'username': GraphQLField(
+            type=GraphQLString,
+            description='Username used for authetication',
+            resolver=lambda root, *_: root.get('username')
+        ),
+        'webCategory': GraphQLField(
+            type=GraphQLString,
+            description='Web content categories',
+            resolver=lambda root, *_: root.get('webcat')
+        ),
+        'referer': GraphQLField(
+            type=GraphQLString,
+            description='The address of the webpage that linked to the resource being requested',
+            resolver=lambda root, *_: root.get('referer')
+        ),
+        'responseCode': GraphQLField(
+            type=GraphQLInt,
+            description='HTTP response code',
+            resolver=lambda root, *_: root.get('respcode') or 0
+        ),
+        'uriPort': GraphQLField(
+            type=GraphQLInt,
+            description='URI port',
+            resolver=lambda root, *_: root.get('uriport')
+        ),
+        'uriPath': GraphQLField(
+            type=GraphQLString,
+            description='URI path',
+            resolver=lambda root, *_: root.get('uripath')
+        ),
+        'uriQuery': GraphQLField(
+            type=GraphQLString,
+            description='URI query',
+            resolver=lambda root, *_: root.get('uriquery')
+        ),
+        'serverIp': GraphQLField(
+            type=SpotIpType,
+            description='Server/Proxy IP',
+            resolver=lambda root, *_: root.get('serverip')
+        ),
+        'serverToClientBytes': GraphQLField(
+            type=GraphQLInt,
+            description='Number of bytes sent from appliance to client',
+            resolver=lambda root, *_: root.get('scbytes')
+        ),
+        'clientToServerBytes': GraphQLField(
+            type=GraphQLInt,
+            description='Number of bytes sent from client to appliance',
+            resolver=lambda root, *_: root.get('csbytes')
+        ),
+        'uri': GraphQLField(
+            type=GraphQLString,
+            description='The original URI requested',
+            resolver=lambda root, *_: root.get('fulluri')
+        ),
+        'score': GraphQLField(
+            type=GraphQLInt,
+            description='Spot ML score value',
+            resolver=lambda root, *_: root.get('ml_score') or 0
+        ),
+        'uriRep': GraphQLField(
+            type=GraphQLString,
+            description='URI reputation metadata',
+            resolver=lambda root, *_: root.get('uri_rep')
+        ),
+        'responseCodeLabel': GraphQLField(
+            type=GraphQLString,
+            description='HTTP response code name',
+            resolver=lambda root, *_: root.get('respcode_name')
+        ),
+        'networkContext': GraphQLField(
+            type=GraphQLString,
+            description='@deprecated',
+            resolver=lambda root, *_: root.get('network_context')
+        )
+    }
+)
+
+EdgeDetailsType = GraphQLObjectType(
+    name='ProxyEdgeDetailsType',
+    fields={
+        'datetime': GraphQLField(
+            type=GraphQLString,
+            description='Start time of the request',
+            resolver=lambda root, *_: '{} {}'.format(root.get('tdate') or '1970-01-01', root.get('time') or '00:00:00')
+        ),
+        'clientIp': GraphQLField(
+            type=SpotIpType,
+            description='Client\'s IP address',
+            resolver=lambda root, *_: root.get('clientip')
+        ),
+        'host': GraphQLField(
+            type=GraphQLString,
+            description='Host name from the client request URL',
+            resolver=lambda root, *_: root.get('host')
+        ),
+        'webCategory': GraphQLField(
+            type=GraphQLString,
+            description='Web content categories',
+            resolver=lambda root, *_: root.get('webcat')
+        ),
+        'responseCode': GraphQLField(
+            type=GraphQLInt,
+            description='HTTP response code',
+            resolver=lambda root, *_: root.get('respcode') or 0
+        ),
+        'responseCodeLabel': GraphQLField(
+            type=GraphQLString,
+            description='HTTP response code name',
+            resolver=lambda root, *_: root.get('respcode_name')
+        ),
+        'requestMethod': GraphQLField(
+            type=GraphQLString,
+            description='HTTP request method',
+            resolver=lambda root, *_: root.get('reqmethod')
+        ),
+        'userAgent': GraphQLField(
+            type=GraphQLString,
+            description='Client\'s user agent',
+            resolver=lambda root, *_: root.get('useragent')
+        ),
+        'responseContentType': GraphQLField(
+            type=GraphQLString,
+            description='HTTP response content type (MIME)',
+            resolver=lambda root, *_: root.get('resconttype')
+        ),
+        'referer': GraphQLField(
+            type=GraphQLString,
+            description='The address of the webpage that linked to the resource being requested',
+            resolver=lambda root, *_: root.get('referer')
+        ),
+        'uriPort': GraphQLField(
+            type=GraphQLInt,
+            description='URI port',
+            resolver=lambda root, *_: root.get('uriport')
+        ),
+        'serverIp': GraphQLField(
+            type=SpotIpType,
+            description='Server/Proxy IP',
+            resolver=lambda root, *_: root.get('serverip')
+        ),
+        'serverToClientBytes': GraphQLField(
+            type=GraphQLInt,
+            description='Number of bytes sent from appliance to client',
+            resolver=lambda root, *_: root.get('scbytes')
+        ),
+        'clientToServerBytes': GraphQLField(
+            type=GraphQLInt,
+            description='Number of bytes sent from client to appliance',
+            resolver=lambda root, *_: root.get('csbytes')
+        ),
+        'uri': GraphQLField(
+            type=GraphQLString,
+            description='The original URI requested',
+            resolver=lambda root, *_: root.get('fulluri')
+        )
+    }
+)
+
+ScoredRequestType = GraphQLObjectType(
+    name='ProxyScoredRequestType',
+    fields={
+        'datetime': GraphQLField(
+            type=SpotDateType,
+            description='Date and time of user score',
+            resolver=lambda root, *_: root.get('tdate') or '1970-01-01'
+        ),
+        'uri': GraphQLField(
+            type=SpotIpType,
+            description='Requested URI',
+            resolver=lambda root, *_: root.get('fulluri')
+        ),
+        'score': GraphQLField(
+            type=GraphQLInt,
+            description='URI risk score value. 1->High, 2->Medium, 3->Low',
+            resolver=lambda root, *_: root.get('uri_sev') or 0
+        )
+    }
+)
+
+CommentType = GraphQLObjectType(
+    name='ProxyCommentType',
+    fields={
+        'uri': GraphQLField(
+            type=GraphQLString,
+            description='High risk URI',
+            resolver=lambda root, *_: root.get('p_threat')
+        ),
+        'title': GraphQLField(
+            type=GraphQLString,
+            description='Threat title',
+            resolver=lambda root, *_: root.get('title')
+        ),
+        'text': GraphQLField(
+            type=GraphQLString,
+            description='Threat description',
+            resolver=lambda root, *_: root.get('text')
+        )
+    }
+)
+
+ThreatsInformationType = GraphQLObjectType(
+    name='ProxyThreatsType',
+    fields={
+        'list': GraphQLField(
+            type=GraphQLList(ScoredRequestType),
+            description='List of URIs that have been scored',
+            args={
+                'date': GraphQLArgument(
+                    type=SpotDateType,
+                    description='A date to use as reference to retrieve the list of scored URI. Defaults to today'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.get_scored_requests(date=args.get('date', date.today()))
+        ),
+        'comments': GraphQLField(
+            type=GraphQLList(CommentType),
+            description='A list of comments about threats',
+            args={
+                'date': GraphQLArgument(
+                    type=SpotDateType,
+                    description='A date to use as reference to retrieve the list of high risk comments. Defaults to today'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.story_board(date=args.get('date', date.today()))
+        )
+    }
+)
+
+ThreatDetailsType = GraphQLObjectType(
+    name='ProxyThreatDetailsType',
+    fields={
+        'datetime': GraphQLField(
+            type=SpotDatetimeType,
+            description='Start time of the request',
+            resolver=lambda root, *_: '{} {}'.format(root.get('p_date') or '1970-01-01', root.get('p_time') or '00:00:00')
+        ),
+        'clientIp': GraphQLField(
+            type=SpotIpType,
+            description='Client\'s IP address',
+            resolver=lambda root, *_: root.get('clientip')
+        ),
+        'username': GraphQLField(
+            type=GraphQLString,
+            description='Username used for authetication',
+            resolver=lambda root, *_: root.get('username')
+        ),
+        'duration': GraphQLField(
+            type=GraphQLInt,
+            description='Connection duration',
+            resolver=lambda root, *_: root.get('duration')
+        ),
+        'uri': GraphQLField(
+            type=GraphQLString,
+            description='The original URI requested',
+            resolver=lambda root, *_: root.get('fulluri')
+        ),
+        'webCategory': GraphQLField(
+            type=GraphQLString,
+            description='Web content categories',
+            resolver=lambda root, *_: root.get('webcat')
+        ),
+        'responseCode': GraphQLField(
+            type=GraphQLInt,
+            description='HTTP response code',
+            resolver=lambda root, *_: root.get('respcode')
+        ),
+        'requestMethod': GraphQLField(
+            type=GraphQLString,
+            description='HTTP request method',
+            resolver=lambda root, *_: root.get('reqmethod')
+        ),
+        'userAgent': GraphQLField(
+            type=GraphQLString,
+            description='Client\'s user agent',
+            resolver=lambda root, *_: root.get('useragent')
+        ),
+        'responseContentType': GraphQLField(
+            type=GraphQLString,
+            description='HTTP response content type (MIME)',
+            resolver=lambda root, *_: root.get('resconttype')
+        ),
+        'referer': GraphQLField(
+            type=GraphQLString,
+            description='The address of the webpage that linked to the resource being requested',
+            resolver=lambda root, *_: root.get('referer')
+        ),
+        'uriPort': GraphQLField(
+            type=GraphQLInt,
+            description='URI port',
+            resolver=lambda root, *_: root.get('uriport')
+        ),
+        'serverIp': GraphQLField(
+            type=SpotIpType,
+            description='The address of the webpage that linked to the resource being requested',
+            resolver=lambda root, *_: root.get('serverip')
+        ),
+        'serverToClientBytes': GraphQLField(
+            type=GraphQLInt,
+            description='Number of bytes sent from appliance to client',
+            resolver=lambda root, *_: root.get('scbytes')
+        ),
+        'clientToServerBytes': GraphQLField(
+            type=GraphQLInt,
+            description='Number of bytes sent from client to appliance',
+            resolver=lambda root, *_: root.get('csbytes')
+        )
+    }
+)
+
+IncidentProgressionRequestType = GraphQLObjectType(
+    name='ProxyIncidentProgressionRequestType',
+    fields={
+        'clientIp': GraphQLField(
+            type=SpotIpType,
+            description='Client\'s IP',
+            resolver=lambda root, *_: root.get('clientip')
+        ),
+        'referer': GraphQLField(
+            type=GraphQLString,
+            description='The address of the webpage that linked to the resource being requested',
+            resolver=lambda root, *_: root.get('referer')
+        ),
+        'requestMethod': GraphQLField(
+            type=GraphQLString,
+            description='HTTP Request Method',
+            resolver=lambda root, *_: root.get('reqmethod')
+        ),
+        'responseContentType': GraphQLField(
+            type=GraphQLString,
+            description='HTTP response content type (MIME)',
+            resolver=lambda root, *_: root.get('resconttype')
+        )
+    }
+)
+
+IncidentProgressionType = GraphQLObjectType(
+    name='ProxyIncidentProgressionType',
+    fields={
+        'uri': GraphQLField(
+            type=GraphQLString,
+            description='Threat URI',
+            resolver=lambda root, *_: root.get('fulluri')
+        ),
+        'refererFor': GraphQLField(
+            type=GraphQLList(GraphQLString),
+            description='A list of URI who whose referer is the threat\'s URI',
+            resolver=lambda root, *_: root.get('referer_for')
+        ),
+        'requests': GraphQLField(
+            type=GraphQLList(IncidentProgressionRequestType),
+            description='A list of requests made to Threat\'s URI',
+            resolver=lambda root, *_: root.get('requests')
+        )
+    }
+)
+
+TimelineType = GraphQLObjectType(
+    name='ProxyTimelineType',
+    fields={
+        'startDatetime': GraphQLField(
+            type=SpotDatetimeType,
+            description='Connection\'s start time',
+            resolver=lambda root, *_: root.get('tstart') or '1970-01-01 00:00:00'
+        ),
+        'endDatetime': GraphQLField(
+            type=SpotDatetimeType,
+            description='Connection\'s end time',
+            resolver=lambda root, *_: root.get('tend') or '1970-01-01 00:00:00'
+        ),
+        'duration': GraphQLField(
+            type=GraphQLInt,
+            description='Connection duration',
+            resolver=lambda root, *_: root.get('duration')
+        ),
+        'clientIp': GraphQLField(
+            type=SpotIpType,
+            description='Client\'s IP address',
+            resolver=lambda root, *_: root.get('clientip')
+        ),
+        'responseCode': GraphQLField(
+            type=GraphQLInt,
+            description='HTTP response code',
+            resolver=lambda root, *_: root.get('respcode')
+        ),
+        'responseCodeLabel': GraphQLField(
+            type=GraphQLString,
+            description='HTTP response code name',
+            resolver=lambda root, *_: root.get('respcode_name')
+        )
+    }
+)
+
+ThreatInformationType = GraphQLObjectType(
+    name='ProxyThreatInformation',
+    fields={
+        'details': GraphQLField(
+            type=GraphQLList(ThreatDetailsType),
+            description='Detailed information about a high risk threat',
+            args={
+                'date': GraphQLArgument(
+                    type=SpotDateType,
+                    description='A date to use as reference for detailed information. Defaults to today'
+                ),
+                'uri': GraphQLArgument(
+                    type=GraphQLNonNull(GraphQLString),
+                    description='Threat\'s URI'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.expanded_search(date=args.get('date', date.today()), uri=args.get('uri'))
+        ),
+        'incidentProgression': GraphQLField(
+            type=IncidentProgressionType,
+            description='Details the type of connections that conform the activity related to the threat',
+            args={
+                'date': GraphQLArgument(
+                    type=SpotDateType,
+                    description='A date to use as reference for incident progression information. Defaults to today'
+                ),
+                'uri': GraphQLArgument(
+                    type=GraphQLNonNull(GraphQLString),
+                    description='Threat URI'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.incident_progression(date=args.get('date', date.today()), uri=args.get('uri'))
+        ),
+        'timeline': GraphQLField(
+            type=GraphQLList(TimelineType),
+            description='Lists \'clusters\' of inbound connections to the IP, grouped by time; showing an overall idea of the times during the day with the most activity',
+            args={
+                'date': GraphQLArgument(
+                    type=SpotDateType,
+                    description='A date to use as reference for time line information. Defaults to today'
+                ),
+                'uri': GraphQLArgument(
+                    type=GraphQLNonNull(GraphQLString),
+                    description='Threat URI'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.time_line(date=args.get('date', date.today()), uri=args.get('uri'))
+        )
+    }
+)
+
+QueryType = GraphQLObjectType(
+    name='ProxyQueryType',
+    fields={
+        'suspicious': GraphQLField(
+            type=GraphQLList(SuspiciousType),
+            description='Proxy suspicious requests',
+            args={
+                'date': GraphQLArgument(
+                    type=SpotDateType,
+                    description='A date to use as reference to retrieve the list of suspicious requests. Defaults to today'
+                ),
+                'uri': GraphQLArgument(
+                    type=GraphQLString,
+                    description='URI of interest'
+                ),
+                'clientIp': GraphQLArgument(
+                    type=SpotIpType,
+                    description='Client\'s ip'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.suspicious_requests(date=args.get('date', date.today()), uri=args.get('uri'), ip=args.get('clientIp'))
+        ),
+        'edgeDetails': GraphQLField(
+            type=GraphQLList(EdgeDetailsType),
+            description='HTTP requests to a particular URI',
+            args={
+                'date': GraphQLArgument(
+                    type=SpotDateType,
+                    description='A date to use as reference to retrieve the list of requests made by client. Defaults to today'
+                ),
+                'uri': GraphQLArgument(
+                    type=GraphQLNonNull(GraphQLString),
+                    description='URI of interest'
+                ),
+                'clientIp': GraphQLArgument(
+                    type=GraphQLNonNull(SpotIpType),
+                    description='Client\'s IP'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.details(date=args.get('date', date.today()), uri=args.get('uri'), ip=args.get('clientIp'))
+        ),
+        'threats': GraphQLField(
+            type=ThreatsInformationType,
+            description='Advanced inforamtion about threats',
+            resolver=lambda *_ : {}
+        ),
+        'threat': GraphQLField(
+            type=ThreatInformationType,
+            description='Advanced inforamtion about a single threat',
+            resolver=lambda *_:{}
+        ),
+        'ingestSummary': GraphQLField(
+            type=GraphQLList(IngestSummaryType),
+            description='Summary of ingested proxy records in range',
+            args={
+                'startDate': GraphQLArgument(
+                    type=GraphQLNonNull(SpotDateType),
+                    description='Start date'
+                ),
+                'endDate': GraphQLArgument(
+                    type=GraphQLNonNull(SpotDateType),
+                    description='End date'
+                )
+            },
+            resolver=lambda root, args, *_: Proxy.ingest_summary(start_date=args.get('startDate'), end_date=args.get('endDate'))
+        )
+    }
+)
+
+TYPES = []

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/schema.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/schema.py b/spot-oa/api/graphql/schema.py
new file mode 100644
index 0000000..324128d
--- /dev/null
+++ b/spot-oa/api/graphql/schema.py
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from graphql import (
+    GraphQLSchema,
+    GraphQLObjectType,
+    GraphQLField
+)
+
+from flow import QueryType as NetflowQueryType, MutationType as NetflowMutationType, TYPES as NetflowTypes
+from dns import QueryType as DnsQueryType, MutationType as DnsMutationType, TYPES as DnsTypes
+from proxy import QueryType as ProxyQueryType, MutationType as ProxyMutationType, TYPES as ProxyTypes
+
+SpotSchema = GraphQLSchema(
+  query=GraphQLObjectType(
+    name='SpotQueryType',
+    fields={
+      'flow': GraphQLField(
+        type=NetflowQueryType,
+        description='Flow is a network protocol that collects IP traffic information and monitors network traffic',
+        resolver=lambda *_: {}
+      ),
+      'dns': GraphQLField(
+        type=DnsQueryType,
+        description='Domain Name System (DNS) Log Records contains the requests in between clients and DNS servers',
+        resolver=lambda *_: {}
+      ),
+      'proxy': GraphQLField(
+        type=ProxyQueryType,
+        description='Proxy Logs contains the requests in between clients and Proxy servers',
+        resolver=lambda *_: {}
+      )
+    }
+  ),
+  mutation=GraphQLObjectType(
+    name='SpotMutationType',
+    fields={
+        'flow': GraphQLField(
+            type=NetflowMutationType,
+            description='Flow related mutation operations',
+            resolver=lambda *_: {}
+        ),
+        'dns': GraphQLField(
+            type=DnsMutationType,
+            description='DNS related mutation operations',
+            resolver=lambda *_: {}
+        ),
+        'proxy': GraphQLField(
+            type=ProxyMutationType,
+            description='Proxy related mutation operations',
+            resolver=lambda *_: {}
+        )
+    }
+  ),
+  types=NetflowTypes + DnsTypes + ProxyTypes
+)

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/graphql/webapp.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/graphql/webapp.py b/spot-oa/api/graphql/webapp.py
new file mode 100644
index 0000000..5a4bd23
--- /dev/null
+++ b/spot-oa/api/graphql/webapp.py
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+if __name__=='__main__':
+    import sys
+    from os import path
+
+    sys.path.append(path.dirname(path.dirname(path.dirname(__file__))))
+
+from flask import Flask, Blueprint
+from flask_graphql import GraphQLView
+import os
+
+from schema import SpotSchema
+
+app = Flask(__name__)
+
+blueprint = Blueprint('graphql_api', __name__)
+blueprint.add_url_rule('/graphql', strict_slashes=False, view_func=GraphQLView.as_view('graphql', schema=SpotSchema, graphiql=os.environ.get('SPOT_DEV')=='1'))
+
+app.register_blueprint(blueprint)
+
+if __name__=='__main__':
+    port = int(sys.argv[1]) if len(sys.argv)>1 else 8889
+
+    app.run(host='0.0.0.0', port=port)
+
+def load_jupyter_server_extension(nb_app):
+    import tornado.web
+    import tornado.wsgi
+
+    wsgi_app = tornado.wsgi.WSGIContainer(app)
+    nb_app.web_app.add_handlers(r'.*', [
+        (r'/graphql.*', tornado.web.FallbackHandler, dict(fallback=wsgi_app))
+    ])
+
+    nb_app.log.info('Apache Spot server extension loaded')
+    if os.environ.get('SPOT_DEV')=='1':
+        nb_app.log.warn('Apache Spot server running in dev mode (environment var SPOT_DEV=1)')

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/README.md
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/README.md b/spot-oa/api/resources/README.md
new file mode 100644
index 0000000..ce8b311
--- /dev/null
+++ b/spot-oa/api/resources/README.md
@@ -0,0 +1,49 @@
+
+
+# API Resources 
+
+
+API Resources are the backend methods used by GraphQL to perform CRUD operations to Apache Spot (incubating) like score connections, perform threat investigation, generate a storyboard, etc.
+
+**Classes:**
+
+* resources/flow.py
+* resources/dns
+* resources/proxy
+* resources/configurator
+* resources/hdfs_client
+* resources/impala_engine.py
+
+
+
+## **Configuration Required (spot.conf):**
+API Resources use [WebHDFS REST API] (https://hadoop.apache.org/docs/r1.0.4/webhdfs.html), based on that spot.conf has been updated to include new KEYS (o variables o entries)
+
+**_Keys in HDFS section:_**
+
+* **NAME_NODE:** this key is required to setup the name node (full DNS domain or IP) to get connected to WebHDFS REST API.
+* **WEB_PORT:** Web port for WebHDFS REST API (default=50070)
+
+**_Keys in Impala section:_**
+
+* **IMPALA_DEM:** This key was part of previous release. Now that spot uses an API to connect, you need to either put the impala daemon full DNS or Server IP Address.
+* **IMPALA_PORT:** Port on which HiveServer2 client requests are served by Impala Daemons.
+
+## **Prerequisites:**
+
+#### Python:
+* setuptools
+* thrift==0.9.3
+* impyla
+* hdfs
+
+**NOTE:** all these requirements are already part of requirements.txt file, you don't need to install the python prerequisites manually. For more information go to [install python requirements](../../README.md)
+
+#### Hadoop:
+
+* Impala.
+* WebHDFS REST API.
+
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/__init__.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/__init__.py b/spot-oa/api/resources/__init__.py
new file mode 100755
index 0000000..ecb1860
--- /dev/null
+++ b/spot-oa/api/resources/__init__.py
@@ -0,0 +1,16 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/configurator.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/configurator.py b/spot-oa/api/resources/configurator.py
new file mode 100644
index 0000000..5bda045
--- /dev/null
+++ b/spot-oa/api/resources/configurator.py
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import ConfigParser
+import os
+
+def configuration():
+
+    conf_file = "/etc/spot.conf"
+    config = ConfigParser.ConfigParser()
+    config.readfp(SecHead(open(conf_file)))
+    return config
+
+def db():
+    conf = configuration()
+    return conf.get('conf', 'DBNAME').replace("'","").replace('"','')
+
+def impala():
+    conf = configuration()
+    return conf.get('conf', 'IMPALA_DEM'),conf.get('conf', 'IMPALA_PORT')
+
+def hdfs():
+    conf = configuration()
+    name_node = conf.get('conf',"NAME_NODE")
+    web_port = conf.get('conf',"WEB_PORT")
+    hdfs_user = conf.get('conf',"HUSER")
+    hdfs_user = hdfs_user.split("/")[-1].replace("'","").replace('"','')
+    return name_node,web_port,hdfs_user
+
+def spot():
+    conf = configuration()
+    return conf.get('conf',"HUSER").replace("'","").replace('"','')
+
+class SecHead(object):
+    def __init__(self, fp):
+        self.fp = fp
+        self.sechead = '[conf]\n'
+
+    def readline(self):
+        if self.sechead:
+            try:
+                return self.sechead
+            finally:
+                self.sechead = None
+        else:
+            return self.fp.readline()

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/dns.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/dns.py b/spot-oa/api/resources/dns.py
new file mode 100644
index 0000000..a9a45d6
--- /dev/null
+++ b/spot-oa/api/resources/dns.py
@@ -0,0 +1,395 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import api.resources.impala_engine as ImpalaEngine
+import api.resources.hdfs_client as HDFSClient
+import api.resources.configurator as Configuration
+from hdfs.util import HdfsError
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the suspicious dns connections in one day.
+--------------------------------------------------------------------------
+"""
+def suspicious_queries(date, ip=None, query=None,limit=250):
+
+    db = Configuration.db()
+    sq_query = ("""
+            SELECT STRAIGHT_JOIN
+                ds.unix_tstamp,frame_len,ds.ip_dst,ds.dns_qry_name,
+                dns_qry_class,dns_qry_type,dns_qry_rcode,ml_score,tld,
+                query_rep,hh,dns_qry_class_name,dns_qry_type_name,
+                dns_qry_rcode_name,network_context
+            FROM {0}.dns_scores ds
+            LEFT JOIN {0}.dns_threat_investigation dt
+                ON  (ds.dns_qry_name = dt.dns_qry_name)
+            WHERE
+                ds.y={1} AND ds.m={2} AND ds.d={3}
+                AND (dt.dns_qry_name is NULL)
+            """).format(db,date.year,date.month,date.day)
+
+    sq_filter = ""
+    sq_filter += " AND ds.ip_dst = '{0}'".format(ip) if ip else ""
+    sq_filter += " AND ds.dns_qry_name LIKE '%{0}%'".format(query) if query else ""
+    sq_filter += " ORDER BY ds.ml_score limit {0}".format(limit)
+
+    sq_query = sq_query + sq_filter
+    return ImpalaEngine.execute_query_as_list(sq_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connectios related to a query name in one hour
+--------------------------------------------------------------------------
+"""
+def details(frame_time, query):
+
+    db = Configuration.db()
+    details_query = ("""
+            SELECT
+		unix_tstamp,frame_len,ip_dst,ip_src,dns_qry_name,dns_qry_class,
+		dns_qry_type,dns_qry_rcode,dns_a,dns_qry_type_name,
+		dns_qry_rcode_name,dns_qry_class_name
+            FROM
+                {0}.dns_edge
+            WHERE
+                y={1} AND m={2} AND d={3} AND hh={4} AND dns_qry_name = '{5}'
+            """).format(db,frame_time.year,frame_time.month,frame_time.day,\
+            frame_time.hour,query)
+
+    return ImpalaEngine.execute_query_as_list(details_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connections with a single client.
+--------------------------------------------------------------------------
+"""
+def client_details(date, ip):
+
+    db = Configuration.db()
+    client_query =("""
+            SELECT
+                ip_dst,dns_a,dns_qry_name,ip_dst
+            FROM
+                {0}.dns_dendro
+            WHERE
+                y={1} AND m={2} AND d={3}
+                AND ip_dst='{4}'
+            """).format(db,date.year,date.month,date.day,ip)
+
+    return ImpalaEngine.execute_query_as_list(client_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the detected suspicious connections in one day.
+--------------------------------------------------------------------------
+"""
+def incident_progression(date, query,ip):
+
+    if not ip and not query:
+        return None
+
+    db = Configuration.db()
+    return_value = "dns_qry_name" if ip else "ip_dst"
+    dns_threat_query = ("""
+            SELECT
+                anchor,total,{0}
+            FROM
+                {1}.dns_threat_dendro
+            WHERE
+                y={2} AND m={3} AND d={4}
+                AND anchor = '{5}'
+            """).format(return_value,db,date.year,date.month,date.day,\
+            query if query else ip)
+                
+    return ImpalaEngine.execute_query_as_list(dns_threat_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all queries saved in threat investigation.
+--------------------------------------------------------------------------
+"""
+def comments(date):
+
+    db = Configuration.db()
+    comments_query = ("""
+            SELECT
+                ip_threat,dns_threat,title,text
+            FROM
+                {0}.dns_storyboard
+            WHERE
+                y={1} AND m={2} AND d={3}
+            """).format(db,date.year,date.month,date.day)
+
+    results = ImpalaEngine.execute_query_as_list(comments_query)
+    for row in results:
+        row["text"] = row["text"].replace("\n","\\n")
+    return results
+
+"""
+--------------------------------------------------------------------------
+Score connections.
+--------------------------------------------------------------------------
+"""
+def  score_connection(date,ip="", dns="", ip_sev=0, dns_sev=0):
+
+    if (not ip and not ip_sev) and (not dns and not dns_sev):
+        return False
+
+    db = Configuration.db()
+    sq_query = ("""
+		SELECT
+    	    frame_time,unix_tstamp,frame_len,ip_dst,dns_qry_name,dns_qry_class,
+		    dns_qry_type,dns_qry_rcode,ml_score,tld,query_rep,
+		    hh,dns_qry_class_name,dns_qry_type_name,dns_qry_rcode_name,
+		    network_context
+		FROM
+		    {0}.dns_scores
+		WHERE
+		    y={1} and m={2} and d={3}
+            AND (
+		""").format(db,date.year,date.month,date.day)
+
+    connections_filter = ""
+    connections_filter += "ip_dst = '{0}' ".format(ip) if ip else ""
+    connections_filter += " OR " if ip and dns else ""
+    connections_filter += "dns_qry_name = '{0}' ".format(dns) if dns else ""
+    connections_filter += ")"
+    connections = ImpalaEngine.execute_query(sq_query + connections_filter)
+
+    # add score to connections
+
+    insert_command = ("""INSERT INTO {0}.dns_threat_investigation
+                        PARTITION (y={1},m={2},d={3})
+                        VALUES (""") \
+                        .format(db,date.year,date.month,date.day)
+
+    fb_data =  []
+    first = True
+    num_rows = 0
+    for row in connections:
+        # insert into dns_threat_investigation.
+        threat_data = (row[1],row[3],row[4],ip_sev if ip == row[3] else 0,\
+        dns_sev if dns == row[4] else 0)
+
+        fb_data.append([row[0],row[2],row[3],row[4],row[5],row[6],row[7],\
+        row[8],row[9],row[10],row[11],ip_sev,dns_sev,row[12],row[13],row[14],\
+        row[15],row[1]])
+
+        insert_command += "{0}{1}".format("," if not first else "", threat_data)
+        first = False
+        num_rows += 1
+
+    insert_command += ")"
+    if num_rows > 0: ImpalaEngine.execute_query(insert_command)
+
+    # create feedback file.
+    app_path = Configuration.spot()
+    feedback_path = "{0}/dns/scored_results/{1}{2}{3}/feedback"\
+    .format(app_path,date.year,str(date.month).zfill(2),str(date.day).zfill(2))
+    ap_file = True
+
+    if len(HDFSClient.list_dir(feedback_path)) == 0:
+        fb_data.insert(0,["frame_time","frame_len","ip_dst","dns_qry_name",\
+        "dns_qry_class","dns_qry_type","dns_qry_rcode","score","tld","query_rep",\
+        "hh","ip_sev","dns_sev","dns_qry_class_name","dns_qry_type_name",\
+        "dns_qry_rcode_name","network_context","unix_tstamp"])
+        ap_file = False
+
+    HDFSClient.put_file_csv(fb_data,feedback_path,"ml_feedback.csv",append_file=ap_file)
+    return True
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the scored connections in one day.
+--------------------------------------------------------------------------
+"""
+def  get_scored_connections(date):
+
+    db = Configuration.db()
+    sc_query =  ("""
+                SELECT
+                    unix_tstamp,ip_dst,dns_qry_name,ip_sev,dns_sev
+                FROM
+                    {0}.dns_threat_investigation
+                WHERE
+                    y={1} AND m={2} AND d={3}
+                """).format(db,date.year,date.month,date.day)
+
+    return ImpalaEngine.execute_query_as_list(sc_query)
+
+"""
+--------------------------------------------------------------------------
+Get expanded search from raw data table.
+--------------------------------------------------------------------------
+"""
+def expanded_search(date,query=None,ip=None,limit=20):
+
+    if not ip and not query:
+        return False
+
+    db = Configuration.db()
+    if ip:
+	count = "dns_qry_name"
+        filter_param = "ip_dst"
+	filter_value = ip
+    else:
+	count = "ip_dst"
+	filter_param = "dns_qry_name"
+	filter_value = query
+
+    expanded_query = ("""
+   		SELECT
+    		    COUNT({0}) as total,dns_qry_name,ip_dst
+		FROM
+		    {1}.dns
+		WHERE y={2} AND m={3} AND d={4}
+		AND {5} = '{6}'
+		GROUP BY {0},{5}
+		ORDER BY total DESC
+		LIMIT {7}
+    """).format(count,db,date.year,date.month,date.day,\
+    filter_param,filter_value,limit if limit else 20)
+
+    return ImpalaEngine.execute_query_as_list(expanded_query)
+
+"""
+--------------------------------------------------------------------------
+Create StoryBoard.
+--------------------------------------------------------------------------
+"""
+def create_storyboard(expanded_search,date,ip,query,title,text):
+
+    if not ip and not query:
+        return False
+
+    anchor = ip if ip else query
+    create_dendro(expanded_search,date,anchor)
+    save_comments(anchor,ip,query,title,text,date)
+
+"""
+--------------------------------------------------------------------------
+Create dendrogram for StoryBoard.
+--------------------------------------------------------------------------
+"""
+def create_dendro(expanded_search,date,anchor):
+
+    db = Configuration.db()
+    for row in expanded_search:
+	dendro_query = ("""
+		INSERT INTO {0}.dns_threat_dendro PARTITION (y={1}, m={2},d={3})
+		VALUES ( '{4}',{5},'{6}','{7}')
+		""")\
+        .format(db,date.year,date.month,date.day,anchor,\
+        row["total"],row["dnsQuery"],row["clientIp"])
+
+	ImpalaEngine.execute_query(dendro_query)
+
+"""
+--------------------------------------------------------------------------
+Create save comments for StoryBoard.
+--------------------------------------------------------------------------
+"""
+def  save_comments(anchor,ip,query,title,text,date):
+
+    db = Configuration.db()
+    sb_query = ("""
+            SELECT
+                ip_threat,dns_threat,title,text
+            FROM
+                {0}.dns_storyboard
+            WHERE
+                y = {1} AND m= {2} AND d={3}
+            """).format(db,date.year,date.month,date.day)
+    sb_data = ImpalaEngine.execute_query_as_list(sb_query)
+
+    # find value if already exists.
+    saved = False
+    for item in sb_data:
+        if item["ip_threat"] == anchor or item["dns_threat"]== anchor:
+            item["title"] = title
+            item["text"] = text
+            saved = True
+
+    if not saved:
+        sb_data.append({'text': text, 'ip_threat': str(ip), 'title': title,'dns_threat':query})
+
+
+    #remove old file.
+    app_path = Configuration.spot()
+    old_file = "{0}/dns/hive/oa/storyboard/y={1}/m={2}/d={3}/"\
+    .format(app_path,date.year,date.month,date.day)
+
+    HDFSClient.delete_folder(old_file,"impala")
+    ImpalaEngine.execute_query("invalidate metadata")
+
+    for item in sb_data:
+	insert_query = ("""
+         	INSERT INTO {0}.dns_storyboard PARTITION(y={1} , m={2} ,d={3})
+            	VALUES ( '{4}', '{5}', '{6}','{7}')
+            	""")\
+                .format(db,date.year,date.month,date.day,\
+                item["ip_threat"],item["dns_threat"],item["title"],item["text"])
+        ImpalaEngine.execute_query(insert_query)
+
+    return True
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with all the data ingested during the timeframe
+provided.
+--------------------------------------------------------------------------
+"""
+def ingest_summary(start_date,end_date):
+
+    db = Configuration.db()
+    is_query = ("""
+                SELECT
+                    tdate,total
+                FROM {0}.dns_ingest_summary
+                WHERE
+                    ( y >= {1} and y <= {2}) AND
+                    ( m >= {3} and m <= {4}) AND
+                    ( d >= {5} and d <= {6})
+                """)\
+                .format(db,start_date.year,end_date.year,start_date.month,end_date.month, start_date.day, end_date.day)
+
+    return ImpalaEngine.execute_query_as_list(is_query)
+
+"""
+--------------------------------------------------------------------------
+Reset scored connections.
+--------------------------------------------------------------------------
+"""
+def reset_scored_connections(date):
+
+    dns_storyboard =  "dns/hive/oa/storyboard"
+    dns_threat_investigation = "dns_threat_dendro/hive/oa/threat_dendro"
+    dns_timeline = "dns/hive/oa/threat_investigation"    
+    app_path = Configuration.spot()   
+
+    try:
+        # remove parquet files manually to allow the comments update.
+        HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+            app_path,dns_storyboard,date.year,date.month,date.day) , "impala")
+        HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+            app_path,dns_threat_investigation,date.year,date.month,date.day), "impala")
+        HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+            app_path,dns_timeline,date.year,date.month,date.day), "impala")
+        ImpalaEngine.execute_query("invalidate metadata")
+        return True
+
+    except HdfsError:
+        return False

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/flow.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/flow.py b/spot-oa/api/resources/flow.py
new file mode 100755
index 0000000..ab5105f
--- /dev/null
+++ b/spot-oa/api/resources/flow.py
@@ -0,0 +1,958 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import api.resources.impala_engine as ImpalaEngine
+import api.resources.hdfs_client as HDFSClient
+import api.resources.configurator as Configuration
+import os,csv
+import linecache,bisect
+import struct, socket
+from hdfs.util import HdfsError
+import json
+import numpy as np
+
+"""
+--------------------------------------------------------------------------
+Return a list (dict) of all the suspicious connections that happened
+during the specified timeframe.
+--------------------------------------------------------------------------
+"""
+def suspicious_connections(date,ip=None,limit=250):
+
+    db = Configuration.db()
+    sc_query = ("""
+                SELECT STRAIGHT_JOIN
+                    fs.tstart,fs.srcip,fs.dstip,fs.sport,fs.dport,proto,
+                    ipkt,ibyt,opkt,obyt,ml_score,rank,srcip_internal,
+                    dstip_internal,src_geoloc,dst_geoloc,src_domain,
+                    dst_domain,src_rep,dst_rep
+                FROM {0}.flow_scores fs
+                LEFT JOIN {0}.flow_threat_investigation ft
+                    ON (( fs.srcip = ft.srcip) OR ( fs.dstip = ft.dstip))
+                WHERE fs.y={1} AND fs.m={2} and fs.d={3}
+                    AND ( ft.srcip is NULL AND ft.dstip is NULL )
+                """).format(db,date.year,date.month,date.day)
+
+    sc_filter = ""
+    if ip:
+        sc_filter = " AND ( fs.srcip='{0}' OR fs.dstip='{0}')".format(ip)
+
+    sc_filter += " ORDER BY rank  limit {0}".format(limit)
+    sc_query = sc_query + sc_filter
+    return ImpalaEngine.execute_query_as_list(sc_query)
+
+"""
+--------------------------------------------------------------------------
+Retuarn a list(dict) of all the connections that happened
+between 2 IPs in one minute.
+--------------------------------------------------------------------------
+"""
+def details(src_ip,dst_ip,date):
+
+    db = Configuration.db()
+    details_query = ("""
+            SELECT
+                tstart,srcip,dstip,sport,dport,proto,flags,
+                tos,ibyt,ipkt,input,output,rip,obyt,opkt
+            FROM {0}.flow_edge
+            WHERE
+                y={1} AND m={2} AND d={3} AND hh={4} AND mn={5}
+                AND ((srcip='{6}' AND dstip='{7}')
+                OR  (srcip='{7}' AND dstip='{6}'))
+            ORDER BY tstart
+            """).format(db,date.year,date.month,date.day,date.hour, \
+                        date.minute,src_ip,dst_ip)
+
+    return ImpalaEngine.execute_query_as_list(details_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of all the connections related with a single IP.
+--------------------------------------------------------------------------
+"""
+def chord_details(ip,date):
+
+    db = Configuration.db()
+    chord_query =  ("""
+            SELECT
+                srcip,dstip,ibyt,ipkt
+            FROM {0}.flow_chords
+            WHERE  y={1} AND m={2} AND d={3} AND ip_threat='{4}'
+            """).format(db,date.year,date.month,date.day,ip)
+
+    return ImpalaEngine.execute_query_as_list(chord_query)
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with all the data ingested during
+the timeframe provided.
+--------------------------------------------------------------------------
+"""
+def ingest_summary(start_date,end_date):
+
+    db = Configuration.db()
+    is_query = ("""
+            SELECT
+                tdate,total
+            FROM {0}.flow_ingest_summary
+            WHERE
+                ( y >= {1} AND y <= {2}) AND
+                ( m >= {3} AND m <= {4}) AND
+                ( d >= {5} AND d <= {6})
+            ORDER BY tdate
+            """).format(db,start_date.year,end_date.year, \
+                        start_date.month,end_date.month, \
+                        start_date.day, end_date.day)
+
+    return ImpalaEngine.execute_query_as_list(is_query)
+
+"""
+--------------------------------------------------------------------------
+Return list(dict) of connecions that happened between 2 ip
+grouped by second.
+--------------------------------------------------------------------------
+"""
+def time_line(ip,date):
+
+    db = Configuration.db()
+    time_line_query = ("""
+        SELECT
+            ip_threat,tstart,tend,srcip,dstip,proto,
+		    sport,dport,ipkt,ibyt
+        FROM {0}.flow_timeline
+        WHERE
+            y={1} AND m={2} AND d={3}
+            AND ip_threat = '{4}'
+        """).format(db,date.year,date.month,date.day,ip)
+
+    return ImpalaEngine.execute_query_as_list(time_line_query)
+
+"""
+--------------------------------------------------------------------------
+Return json file with all the  geo localization information.
+--------------------------------------------------------------------------
+"""
+def sc_geo(ip,date):
+
+    app_path = Configuration.spot()
+    file_name = "globe-{0}.json".format(ip.replace(".","_"))
+    hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+    .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+    if HDFSClient.file_exists(hdfs_path,file_name):
+        return json.loads(HDFSClient.get_file("{0}/{1}" \
+        .format(hdfs_path,file_name)))
+    else:
+        return {}
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with the ip threatn information captured by the
+security expert.
+--------------------------------------------------------------------------
+"""
+def story_board(date):
+
+    db = Configuration.db()
+    sb_query= ("""
+            SELECT
+                ip_threat,title,text
+            FROM
+                {0}.flow_storyboard
+            WHERE
+                y={1} AND m={2} AND d={3}
+            """).format(db,date.year,date.month,date.day)
+
+    results = ImpalaEngine.execute_query_as_list(sb_query)
+    for row in results:
+	       row["text"] = row["text"].replace("\n","\\n")
+    return results
+
+"""
+--------------------------------------------------------------------------
+Return a json file with the impact analysis information.
+--------------------------------------------------------------------------
+"""
+def impact_analysis(ip,date):
+
+    app_path = Configuration.spot()
+    file_name = "stats-{0}.json".format(ip.replace(".","_"))
+    hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+    .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+    if HDFSClient.file_exists(hdfs_path,file_name):
+        return json.loads(HDFSClient.get_file("{0}/{1}" \
+        .format(hdfs_path,file_name)))
+    else:
+        return {}
+
+"""
+--------------------------------------------------------------------------
+Return a list(dict) with all the inbound, outbound and twoway connections.
+--------------------------------------------------------------------------
+"""
+def incident_progression(ip,date):
+
+    app_path = Configuration.spot()
+    file_name = "threat-dendro-{0}.json".format(ip.replace(".","_"))
+
+    hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+    .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+    if HDFSClient.file_exists(hdfs_path,file_name):
+        return json.loads(HDFSClient.get_file("{0}/{1}" \
+        .format(hdfs_path,file_name)))
+    else:
+        return {}
+
+"""
+--------------------------------------------------------------------------
+Save scored connection into Threat investigation table.
+--------------------------------------------------------------------------
+"""
+def score_connection(score,date,src_ip=None,dst_ip=None,src_port=None,dst_port=None):
+
+    if not src_ip and not dst_ip and not src_port and not dst_port:
+        return False
+
+    db = Configuration.db()
+    # get connections to score
+    connections_query = ("""
+            SELECT
+                tstart,srcip,dstip,sport,dport, ibyt,ipkt
+            FROM {0}.flow_scores
+            WHERE
+                y = {1} AND m={2} AND d={3}
+            """).format(db,date.year,date.month,date.day)
+
+    connections_filter = ""
+    connections_filter += " AND srcip = '{0}'".format(src_ip) if src_ip else ""
+    connections_filter += " AND dstip = '{0}'".format(dst_ip) if dst_ip else ""
+
+    connections_filter += " AND sport = {0}" \
+    .format(str(src_port)) if src_port else ""
+
+    connections_filter += " AND dport = {0}" \
+    .format(str(dst_port)) if dst_port else ""
+    connections = ImpalaEngine.execute_query(connections_query + connections_filter)
+
+
+    # add score to connections
+    insert_command = ("""
+        INSERT INTO {0}.flow_threat_investigation
+        PARTITION (y={1},m={2},d={3})
+        VALUES (""") \
+        .format(db,date.year,date.month,date.day)
+
+    fb_data =  []
+    first = True
+    num_rows = 0
+    for row in connections:
+        # insert into flow_threat_investigation.
+        threat_data = (row[0],row[1],row[2],row[3],row[4],score)
+        fb_data.append([score,row[0],row[1],row[2],row[3],row[4],row[5],row[6]])
+        insert_command += "{0}{1}".format("," if not first else "", threat_data)
+        first = False
+        num_rows += 1
+
+    insert_command += ")"
+    if num_rows > 0: ImpalaEngine.execute_query(insert_command)
+
+    # create feedback file.
+    app_path = Configuration.spot()
+    feedback_path = "{0}/flow/scored_results/{1}{2}{3}/feedback" \
+    .format(app_path,date.year,str(date.month).zfill(2),str(date.day).zfill(2))
+
+    append_file = True
+    if len(HDFSClient.list_dir(feedback_path)) == 0:
+        fb_data.insert(0,["sev","tstart","sip","dip","sport","dport","ipkt","ibyt"])
+        append_file = False
+
+    HDFSClient.put_file_csv(fb_data,feedback_path,"ml_feedback.csv",\
+    append_file=append_file)
+    return True
+
+"""
+--------------------------------------------------------------------------
+Save connections details to flow_storyboard table.
+--------------------------------------------------------------------------
+"""
+def save_comment(ip,title,text,date):
+
+    #Get current table info.
+    db = Configuration.db()
+    sb_query = ("""
+            SELECT
+                ip_threat,title,text
+            FROM
+                {0}.flow_storyboard
+            WHERE
+                y = {1} AND m= {2} AND d={3}
+            """).format(db,date.year,date.month,date.day)
+
+    sb_data = ImpalaEngine.execute_query_as_list(sb_query)
+
+    # find value if already exists.
+    saved = False
+    for item in sb_data:
+        if item["ip_threat"] == ip:
+            item["title"] = title
+            item["text"] = text
+            saved = True
+
+    if not saved:
+        sb_data.append({'text': text, 'ip_threat': str(ip), 'title': title})
+
+    #remove old file.
+    app_path = Configuration.spot()
+    old_file = "{0}/flow/hive/oa/storyboard/y={1}/m={2}/d={3}/" \
+    .format(app_path,date.year,date.month,date.day)
+
+    # remove file manually to allow the comments update.
+    HDFSClient.delete_folder(old_file,"impala")
+    ImpalaEngine.execute_query("invalidate metadata")
+
+    for item in sb_data:
+	insert_query = ("""
+         	INSERT INTO {0}.flow_storyboard PARTITION(y={1} , m={2} ,d={3})
+            	VALUES ( '{4}', '{5}','{6}')
+            	""") \
+                .format(db,date.year,date.month,date.day, \
+                item["ip_threat"],item["title"],item["text"])
+
+        ImpalaEngine.execute_query(insert_query)
+    return True
+
+"""
+--------------------------------------------------------------------------
+Get scored connections from threat investigation table.
+--------------------------------------------------------------------------
+"""
+def get_scored_connections(date):
+
+
+    db = Configuration.db()
+    scored_query = ("""
+            SELECT
+                tstart,srcip,dstip,srcport,dstport,score
+            FROM
+                {0}.flow_threat_investigation
+            WHERE
+                y={1} AND m={2} AND d={3}
+            """).format(db,date.year,date.month,date.day)
+
+    return ImpalaEngine.execute_query_as_list(scored_query)
+
+"""
+--------------------------------------------------------------------------
+Get expanded search data from raw data table.
+--------------------------------------------------------------------------
+"""
+def expanded_search(date,ip):
+
+    db = Configuration.db()
+    expanded_query = ("""
+		SELECT
+		    min(treceived) as firstseen, max(treceived) as lastseen,
+            sip as srcip, dip as dstip, sport as sport,
+            dport as dport, count(sip) as conns, max(ipkt) as maxpkts,
+		    avg(ipkt) as avgpkts, max(ibyt) as maxbyts, avg(ibyt) as avgbyts
+	    FROM
+		    {0}.flow
+        WHERE
+	        y={1} AND m={2} AND d={3}
+        AND (sip ='{4}'  OR dip='{4}')
+        GROUP BY
+		    sip, dip,sport,dport
+		""").format(db,date.year,date.month,date.day,ip)
+
+    return ImpalaEngine.execute_query_as_list(expanded_query)
+
+"""
+--------------------------------------------------------------------------
+Generates all the required data for StoryBoard.
+--------------------------------------------------------------------------
+"""
+def create_storyboard(expanded_search,date,ip,title,text,top_results=20):
+
+
+    cpath = "{0}/context/" \
+    .format(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+    iploc = "{0}/{1}".format(cpath,'iploc.csv')
+    nwloc = "{0}/{1}".format(cpath,'networkcontext_1.csv')
+
+    connections = get_in_out_and_twoway_conns(expanded_search,top_results)
+    inbound,outbound,twoway = add_network_context(nwloc,connections["inbound"] \
+    ,connections["outbound"],connections["twoway"])
+
+    inbound,outbound,twoway = add_geospatial_info(iploc,inbound,outbound,twoway)
+    create_impact_analysis(ip, inbound,outbound,twoway, "",date)
+    create_map_view(ip,inbound,outbound,twoway,date,iploc)
+    create_incident_progression(ip,inbound,outbound,twoway,date)
+    create_time_line(ip,inbound,outbound,twoway,date)
+    save_comment(ip,title,text,date)
+    return True
+
+"""
+--------------------------------------------------------------------------
+Calculate number of inbound only, two-way, and outbound only.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_in_out_and_twoway_conns(expanded_search,top_results=20):
+
+    inbound = {}
+    outbound = {}
+    twoway = {}
+    srcdict = {}
+    dstdict = {}
+    conns_dict= {}
+    rowct = 0
+
+    for row in expanded_search:
+        srcdict[row['srcIp']] = {
+            'ip_int': struct.unpack("!L", socket.inet_aton(str(row['srcIp'])))[0],
+            'dst_ip': row['dstIp'],
+            'dst_ip_int': struct.unpack("!L", socket.inet_aton(str(row['dstIp'])))[0],
+            'conns': int(row['connections']),
+            'maxbytes': int(row['maxBytes'])
+        }
+        dstdict[row['dstIp']] = {
+            'ip_int': struct.unpack("!L", socket.inet_aton(str(row['dstIp'])))[0],
+            'src_ip': row['srcIp'],
+            'src_ip_int': struct.unpack("!L", socket.inet_aton(str(row['srcIp'])))[0],
+            'conns': int(row['connections']),
+            'maxbytes': int(row['maxBytes'])
+        }
+        rowct +=1
+
+    if rowct > 0:
+        for result in srcdict:
+            if result in dstdict:
+                twoway[result] = srcdict[result]
+            else:
+                outbound[result] = srcdict[result]
+
+        for result in dstdict:
+            if result not in srcdict:
+                inbound[result] = dstdict[result]
+
+    top_inbound_b = {}
+    top_outbound_b = {}
+    top_twoway_b = {}
+
+    if len(inbound) > 0:
+        top_inbound_b = get_top_bytes(inbound,top_results)
+        top_inbound_conns = get_top_conns(inbound,top_results)
+        top_inbound_b.update(top_inbound_conns) # merge the two dictionaries
+    if len(outbound) > 0:
+        top_outbound_b = get_top_bytes(outbound,top_results)
+        top_outbound_conns = get_top_conns(outbound,top_results)
+        top_outbound_b.update(top_outbound_conns) # merge the two dictionaries
+    if len(twoway) > 0:
+        top_twoway_b = get_top_bytes(twoway,top_results)
+        top_twoway_conns = get_top_conns(twoway,top_results)
+        top_twoway_b.update(top_twoway_conns) # merge the two dictionaries
+
+
+    result = {}
+    result["inbound"] = top_inbound_b
+    result["outbound"] = top_outbound_b
+    result["twoway"] = top_twoway_b
+
+    return result
+
+"""
+--------------------------------------------------------------------------
+Create incident progression file.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_incident_progression(anchor, inbound, outbound, twoway, date):
+
+    dendro_fpath = 'threat-dendro-' + anchor.replace('.','_') + ".json"
+    obj = {
+        'name':anchor,
+        'children': [],
+        'time': ""
+    }
+
+    #----- Add Inbound Connections-------#
+    if len(inbound) > 0:
+        obj["children"].append({'name': 'Inbound Only', 'children': [], 'impact': 0})
+        in_ctxs = {}
+        for ip in inbound:
+            if 'nwloc' in inbound[ip] and len(inbound[ip]['nwloc']) > 0:
+                ctx = inbound[ip]['nwloc'][2]
+                if ctx not in in_ctxs:
+                    in_ctxs[ctx] = 1
+                else:
+                    in_ctxs[ctx] += 1
+        for ctx in in_ctxs:
+            obj["children"][0]['children'].append({
+                    'name': ctx,
+                    'impact': in_ctxs[ctx]
+                })
+
+    #------ Add Outbound ----------------#
+    if len(outbound) > 0:
+        obj["children"].append({'name':'Outbound Only','children':[],'impact':0})
+        out_ctxs = {}
+        for ip in outbound:
+            if 'nwloc' in outbound[ip] and len(outbound[ip]['nwloc']) > 0:
+                ctx = outbound[ip]['nwloc'][2]
+                if ctx not in out_ctxs:
+                    out_ctxs[ctx] = 1
+                else:
+                    out_ctxs[ctx] += 1
+        for ctx in out_ctxs:
+            obj["children"][1]['children'].append({
+                    'name': ctx,
+                    'impact': out_ctxs[ctx]
+                })
+
+    #------ Add TwoWay ----------------#
+    if len(twoway) > 0:
+        obj["children"].append({'name':'two way','children': [], 'impact': 0})
+        tw_ctxs = {}
+        for ip in twoway:
+            if 'nwloc' in twoway[ip] and len(twoway[ip]['nwloc']) > 0:
+                ctx = twoway[ip]['nwloc'][2]
+                if ctx not in tw_ctxs:
+                    tw_ctxs[ctx] = 1
+                else:
+                    tw_ctxs[ctx] += 1
+
+        for ctx in tw_ctxs:
+            obj["children"][2]['children'].append({
+                    'name': ctx,
+                    'impact': tw_ctxs[ctx]
+                })
+
+    app_path = Configuration.spot()
+    hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+    .format(app_path,date.year,date.month,date.day,anchor.replace(".","_"))
+
+    if HDFSClient.put_file_json(obj,hdfs_path,dendro_fpath,overwrite_file=True):
+        return "Incident progression successfully created \n"
+    else:
+        return "Incident progression couldn't be created \n"
+
+"""
+--------------------------------------------------------------------------
+Create map view for StoryBoard.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_map_view(ip, inbound, outbound, twoway,date,iploc):
+
+    iplist = ''
+    globe_fpath = 'globe-' + ip.replace('.','_') + ".json"
+    if os.path.isfile(iploc):
+        iplist = np.loadtxt(iploc,dtype=np.uint32,delimiter=',',usecols={0},\
+        converters={0: lambda s: np.uint32(s.replace('"',''))})
+    else:
+        print "No iploc.csv file was found, Map View map won't be created"
+
+    response = ""
+    if iplist != '':
+        
+        globe_json = {}
+        globe_json['type'] = "FeatureCollection"
+        globe_json['sourceips'] = []
+        globe_json['destips'] = []
+        for srcip in twoway:
+            try:
+                row =  twoway[srcip]['geo']
+                globe_json['destips'].append({
+                        'type': 'Feature',
+                        'properties': {
+                            'location':row[8],
+                            'ip':srcip,
+                            'type':1
+                        },
+                        'geometry': {
+                            'type': 'Point',
+                            'coordinates': [float(row[7]), float(row[6])]
+                        }
+                    })
+            except ValueError:
+                pass
+        for dstip in outbound:
+            try:
+                row =  outbound[dstip]['geo']
+                dst_geo = outbound[dstip]['geo_dst']
+                globe_json['sourceips'].append({
+                        'type': 'Feature',
+                        'properties': {
+                            'location':row[8],
+                            'ip':dstip,
+                            'type':3
+                        },
+                        'geometry': {
+                            'type': 'Point',
+                            'coordinates': [float(row[7]), float(row[6])]
+                        }
+                    })
+                globe_json['destips'].append({
+                        'type': 'Feature',
+                        'properties': {
+                            'location':row[8],
+                            'ip':outbound[dstip]['dst_ip'],
+                            'type':3
+                        },
+                        'geometry': {
+                            'type': 'Point',
+                            'coordinates': [float(dst_geo[7]), float(dst_geo[6])]
+                        }
+                    })
+            except ValueError:
+                pass
+        for dstip in inbound:
+            try:
+                row =  inbound[dstip]['geo']
+                dst_geo = inbound[dstip]['geo_src']
+                globe_json['sourceips'].append({
+                        'type': 'Feature',
+                        'properties': {
+                            'location':row[8],
+                            'ip':dstip,
+                            'type':2
+                        },
+                        'geometry': {
+                            'type': 'Point',
+                            'coordinates': [float(row[7]), float(row[6])]
+                        }
+                    })
+                globe_json['destips'].append({
+                        'type': 'Feature',
+                        'properties': {
+                            'location':row[8],
+                            'ip':inbound[dstip]['src_ip'],
+                            'type':2
+                        },
+                        'geometry': {
+                            'type': 'Point',
+                            'coordinates': [float(dst_geo[7]), float(dst_geo[6])]
+                        }
+                    })
+            except ValueError:
+                pass
+        json_str = json.dumps(globe_json)
+        app_path = Configuration.spot()
+        hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+        .format(app_path,date.year,date.month,date.day,ip.replace(".","_"))
+
+        if HDFSClient.put_file_json(globe_json,hdfs_path,globe_fpath,overwrite_file=True) :
+            response = "Geolocation map successfully created \n"
+        else:
+            response = "The map can't be created without an iploc file \n"
+
+    return response
+
+"""
+--------------------------------------------------------------------------
+Create timeline for storyboard.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_time_line(anchor,inbound, outbound, twoway,date):
+
+    top_keys = []
+    if len(twoway) > 0: top_keys.extend(twoway.keys())
+    if len(outbound) > 0: top_keys.extend(outbound.keys())
+    if len(inbound) > 0: top_keys.extend(inbound.keys())
+
+
+    db = Configuration.db()
+
+    imp_query =("""
+        INSERT INTO TABLE {0}.flow_timeline PARTITION (y={4}, m={5},d={6})
+        SELECT
+            '{7}' ,min(treceived) as tstart, max(treceived) as tend,
+            sip as srcIP,dip as dstip, proto as proto, sport as sport,
+            dport AS dport, ipkt as ipkt, ibyt as ibyt
+        FROM
+            {0}.flow
+        WHERE y={4} AND m={5} AND d={6}
+        AND ((dip IN({1}) AND sip ='{2}') OR (sip IN({1}) AND dip ='{2}'))
+        GROUP BY sip, dip, proto, sport, dport, ipkt, ibyt
+        ORDER BY tstart
+        LIMIT {3}
+    """)
+
+    ips = "'" + "','".join(top_keys) + "'"
+    imp_query = imp_query.format(db,ips,anchor,1000,date.year,date.month, date.day,anchor)
+
+    if ImpalaEngine.execute_query(imp_query):
+        return "Timeline successfully created \n"
+    else:
+        return "Timeline couldn't be created \n"
+
+"""
+--------------------------------------------------------------------------
+Create Impact Analysis for StoryBoard.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def create_impact_analysis(anchor, inbound, outbound, twoway, threat_name,date):
+
+    stats_fpath = 'stats-' + anchor.replace('.','_') + ".json"
+
+    obj = {
+        'name':threat_name,
+        'children': [],
+        'size': len(inbound) + len(outbound) + len(twoway)
+    }
+
+    #----- Add Inbound Connections-------#
+    obj["children"].append({'name': 'Inbound Only', 'children': [], 'size': len(inbound)})
+    in_ctxs = {}
+    for ip in inbound:
+        full_ctx = ''
+        if 'nwloc' in inbound[ip] and len(inbound[ip]['nwloc']) > 0:
+            full_ctx = inbound[ip]['nwloc'][2].split('.')[0]
+        ctx = get_ctx_name(full_ctx)
+        if ctx not in in_ctxs:
+            in_ctxs[ctx] = 1
+        else:
+            in_ctxs[ctx] += 1
+    for ctx in in_ctxs:
+        obj["children"][0]['children'].append({
+                'name': ctx,
+                'size': in_ctxs[ctx]
+            })
+
+
+    #------ Add Outbound ----------------#
+    obj["children"].append({'name':'Outbound Only','children':[],'size':len(outbound)})
+    out_ctxs = {}
+    for ip in outbound:
+        full_ctx = ''
+        if 'nwloc' in outbound[ip] and len(outbound[ip]['nwloc']) > 0:
+            full_ctx = outbound[ip]['nwloc'][2].split('.')[0]
+        ctx = get_ctx_name(full_ctx)
+        if ctx not in out_ctxs:
+            out_ctxs[ctx] = 1
+        else:
+            out_ctxs[ctx] += 1
+    for ctx in out_ctxs:
+        obj["children"][1]['children'].append({
+                'name': ctx,
+                'size': out_ctxs[ctx]
+            })
+
+    #------ Add Twoway ----------------#
+    obj["children"].append({'name': 'two way', 'children': [], 'size': len(twoway)})
+    tw_ctxs = {}
+    for ip in twoway:
+        full_ctx = ''
+        if 'nwloc' in twoway[ip] and len(twoway[ip]['nwloc']) > 0:
+            full_ctx = twoway[ip]['nwloc'][2].split('.')[0]
+        ctx = get_ctx_name(full_ctx)
+        if ctx not in tw_ctxs:
+            tw_ctxs[ctx] = 1
+        else:
+            tw_ctxs[ctx] += 1
+
+    for ctx in tw_ctxs:
+        obj["children"][2]['children'].append({
+                'name': ctx,
+                'size': tw_ctxs[ctx]
+            })
+
+    app_path = Configuration.spot()
+    hdfs_path = "{0}/flow/oa/storyboard/{1}/{2}/{3}/{4}" \
+    .format(app_path,date.year,date.month,date.day,anchor.replace(".","_"))
+
+    data = json.dumps(obj)
+    if HDFSClient.put_file_json(obj,hdfs_path,stats_fpath,overwrite_file=True):
+        return "Stats file successfully created \n"
+    else:
+        return "Stats file couldn't be created \n"
+
+"""
+--------------------------------------------------------------------------
+Get topbytess list.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_top_bytes(conns_dict, top):
+    topbytes = sorted(conns_dict.iteritems(),key=lambda (x,y): y['maxbytes'],reverse=True)
+    topbytes = topbytes[0:top]
+    return dict(topbytes)
+
+"""
+--------------------------------------------------------------------------
+Get top connections.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_top_conns(conns_dict, top):
+    topconns = sorted(conns_dict.iteritems(), key=lambda (x,y): y['conns'], reverse=True)
+    topconns = topconns[0:top]
+    return dict(topconns)
+
+"""
+--------------------------------------------------------------------------
+Get network context - get start and end ranges.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def add_network_context(nwloc,inbound,outbound,twoway):
+    nwdict = {}
+    if os.path.isfile(nwloc) :
+        with open(nwloc, 'r') as f:
+            reader = csv.reader(f,delimiter=',')
+            reader.next()
+            #address range, description
+            for row in reader:
+
+                if '/' in row[0]:
+                    #Range in subnet
+                    iprange = row[0].split('/')
+                    if len(iprange) < 2:
+                        ipend = 0
+                    else:
+                        ipend = int(iprange[1])
+                    nwdict[row[0]] = [struct.unpack("!L", \
+                    socket.inet_aton(iprange[0]))[0],\
+                    struct.unpack("!L",socket.inet_aton(iprange[0]))[0]+2**(32-ipend)-1, row[1]]
+                elif '-' in row[0]:
+                    #IP Range
+                    iprange = row[0].split('-')
+                    nwdict[row[0]] = [struct.unpack("!L",\
+                    socket.inet_aton(iprange[0].replace(" ", "")))[0],\
+                    struct.unpack("!L", socket.inet_aton(iprange[1].replace(" ", "")))[0], row[1]]
+                else:
+                    #Exact match
+                    nwdict[row[0]] = [struct.unpack("!L",\
+                    socket.inet_aton(row[0]))[0],struct.unpack("!L",\
+                    socket.inet_aton(row[0]))[0], row[1]]
+
+        for srcip in outbound:
+            temp_ip = struct.unpack("!L", socket.inet_aton(srcip))[0]
+            if srcip in nwdict:
+                inbound[srcip]['nwloc'] = nwdict[srcip]
+            else:
+                matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]
+                outbound[srcip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''
+
+        for dstip in twoway:
+            temp_ip = struct.unpack("!L", socket.inet_aton(dstip))[0]
+            if dstip in nwdict:
+                twoway[dstip]['nwloc'] = nwdict[dstip]
+            else:
+                matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]
+                twoway[dstip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''
+
+        for srcip in inbound:
+            temp_ip = struct.unpack("!L", socket.inet_aton(srcip))[0]
+            if srcip in nwdict:
+                inbound[srcip]['nwloc'] = nwdict[srcip]
+            else:
+                matchingVals = [x for x in nwdict if nwdict[x][1] >= temp_ip and nwdict[x][0] <= temp_ip]
+                inbound[srcip]['nwloc'] = nwdict[matchingVals[0]] if len(matchingVals) > 0 else ''
+
+    return inbound,outbound,twoway
+
+
+"""
+--------------------------------------------------------------------------
+Add Geo spatial info
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def add_geospatial_info(iploc,inbound,outbound,twoway):
+    iplist = ''
+    if os.path.isfile(iploc):
+        iplist = np.loadtxt(iploc,dtype=np.uint32,delimiter=',',usecols={0},\
+        converters={0: lambda s: np.uint32(s.replace('"',''))})
+    else:
+        print "No iploc.csv file was found, Map View map won't be created"
+
+
+    # get geospatial info, only when iplocation file is available
+    if iplist != '':
+        for srcip in outbound:
+            reader = csv.reader([linecache.getline(\
+            iploc, bisect.bisect(iplist,outbound[srcip]['ip_int'])).replace('\n','')])
+
+            outbound[srcip]['geo'] = reader.next()
+            reader = csv.reader([linecache.getline(\
+            iploc, bisect.bisect(iplist,outbound[srcip]['dst_ip_int'])).replace('\n','')])
+            outbound[srcip]['geo_dst'] = reader.next()
+
+        for dstip in twoway:
+            reader = csv.reader([linecache.getline(\
+            iploc,bisect.bisect(iplist,twoway[dstip]['ip_int'])).replace('\n','')])
+            twoway[dstip]['geo'] = reader.next()
+
+        for srcip in inbound:
+            reader = csv.reader([linecache.getline(\
+            iploc, bisect.bisect(iplist,inbound[srcip]['ip_int'])).replace('\n','')])
+
+            inbound[srcip]['geo'] = reader.next()
+            reader = csv.reader([linecache.getline(\
+            iploc, bisect.bisect(iplist,inbound[srcip]['src_ip_int'])).replace('\n','')])
+            inbound[srcip]['geo_src'] = reader.next()
+
+    return inbound,outbound,twoway
+
+"""
+--------------------------------------------------------------------------
+Get context name.
+Migrated from IPython NoteBooks.
+--------------------------------------------------------------------------
+"""
+def get_ctx_name(full_context):
+    ctx= 'DMZ'
+    if "VPN" in full_context:
+        ctx = "VPN"
+    elif "DMZ" in full_context:
+        ctx = "DMZ"
+    elif "Proxy" in full_context:
+        ctx = "Proxy"
+    elif "FW" in full_context:
+        ctx = "FW"
+    return ctx
+
+"""
+--------------------------------------------------------------------------
+Reset scored connections.
+--------------------------------------------------------------------------
+"""
+def reset_scored_connections(date):
+
+    flow_storyboard =  "flow/hive/oa/storyboard"
+    flow_threat_investigation = "flow/hive/oa/threat_investigation"
+    flow_timeline = "flow/hive/oa/timeline"    
+    app_path = Configuration.spot()   
+
+    try:
+        # remove parquet files manually to allow the comments update.
+        HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+            app_path,flow_storyboard,date.year,date.month,date.day) , "impala")
+        HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+            app_path,flow_threat_investigation,date.year,date.month,date.day), "impala")
+        HDFSClient.delete_folder("{0}/{1}/y={2}/m={3}/d={4}/".format( \
+            app_path,flow_timeline,date.year,date.month,date.day), "impala")
+        ImpalaEngine.execute_query("invalidate metadata")
+        return True
+        
+    except HdfsError:
+        return False

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/hdfs_client.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/hdfs_client.py b/spot-oa/api/resources/hdfs_client.py
new file mode 100644
index 0000000..31c5eba
--- /dev/null
+++ b/spot-oa/api/resources/hdfs_client.py
@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from hdfs import InsecureClient
+from hdfs.util import HdfsError
+from json import dump
+import api.resources.configurator as Config
+
+
+def _get_client(user=None):
+    hdfs_nm,hdfs_port,hdfs_user = Config.hdfs()
+    client = InsecureClient('http://{0}:{1}'.format(hdfs_nm,hdfs_port), user= user if user else hdfs_user)
+    return client
+
+def get_file(hdfs_file):
+    client = _get_client()
+    with client.read(hdfs_file) as reader:
+        results = reader.read()
+        return results
+
+def put_file_csv(hdfs_file_content,hdfs_path,hdfs_file_name,append_file=False,overwrite_file=False):
+    
+    try:
+        client = _get_client()
+        hdfs_full_name = "{0}/{1}".format(hdfs_path,hdfs_file_name)
+        with client.write(hdfs_full_name,append=append_file,overwrite=overwrite_file) as writer:
+            for item in hdfs_file_content:
+                data = ','.join(str(d) for d in item)
+                writer.write("{0}\n".format(data))
+        return True
+        
+    except HdfsError:
+        return False
+
+def put_file_json(hdfs_file_content,hdfs_path,hdfs_file_name,append_file=False,overwrite_file=False):
+    
+    try:
+        client = _get_client()
+        hdfs_full_name = "{0}/{1}".format(hdfs_path,hdfs_file_name)
+        with client.write(hdfs_full_name,append=append_file,overwrite=overwrite_file,encoding='utf-8') as writer:
+	        dump(hdfs_file_content, writer)
+        return True
+    except HdfsError:
+        return False
+    
+
+def delete_folder(hdfs_file,user=None):
+    client = _get_client(user)
+    client.delete(hdfs_file,recursive=True)
+
+def list_dir(hdfs_path):
+    try:
+        client = _get_client()
+        return client.list(hdfs_path)
+    except HdfsError:
+        return {}
+
+def file_exists(hdfs_path,file_name):
+    files = list_dir(hdfs_path)
+    if str(file_name) in files:
+	    return True
+    else:
+        return False

http://git-wip-us.apache.org/repos/asf/incubator-spot/blob/fbcfa683/spot-oa/api/resources/impala_engine.py
----------------------------------------------------------------------
diff --git a/spot-oa/api/resources/impala_engine.py b/spot-oa/api/resources/impala_engine.py
new file mode 100644
index 0000000..b7d0148
--- /dev/null
+++ b/spot-oa/api/resources/impala_engine.py
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from impala.dbapi import connect
+import api.resources.configurator as Config
+
+def create_connection():
+
+    impala_host, impala_port =  Config.impala()
+    db = Config.db()
+    conn = connect(host=impala_host, port=int(impala_port),database=db)
+    return conn.cursor()
+
+def execute_query(query,fetch=False):
+
+    impala_cursor = create_connection()
+    impala_cursor.execute(query)
+
+    return impala_cursor if not fetch else impala_cursor.fetchall()
+
+def execute_query_as_list(query):
+
+    query_results = execute_query(query)
+    row_result = {}
+    results = []
+
+    for row in query_results:
+        x=0
+        for header in query_results.description:
+            row_result[header[0]] = row[x]
+            x +=1
+        results.append(row_result)
+        row_result = {}
+
+    return results
+
+